diff --git a/build.gradle b/build.gradle index 812b93ad..14c385b8 100644 --- a/build.gradle +++ b/build.gradle @@ -9,13 +9,11 @@ buildscript { dependencies { classpath 'com.android.tools.build:gradle:8.0.2' classpath 'gradle.plugin.at.zierler:yaml-validator-plugin:1.5.0' - classpath("io.objectbox:objectbox-gradle-plugin:3.7.1") } } apply plugin: 'com.android.application' apply plugin: 'at.zierler.yamlvalidator' -apply plugin: "io.objectbox" apply from: 'gradle/scripts/constants.gradle' apply from: 'gradle/scripts/dictionary-tools.gradle' diff --git a/objectbox-models/default.json b/objectbox-models/default.json deleted file mode 100644 index 8e995d8a..00000000 --- a/objectbox-models/default.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "_note1": "KEEP THIS FILE! Check it into a version control system (VCS) like git.", - "_note2": "ObjectBox manages crucial IDs for your object model. See docs for details.", - "_note3": "If you have VCS merge conflicts, you must resolve them according to ObjectBox docs.", - "entities": [ - { - "id": "1:8686759156275895115", - "lastPropertyId": "9:148562041880145406", - "name": "Word", - "properties": [ - { - "id": "1:7053654361616810150", - "name": "id", - "type": 6, - "flags": 1 - }, - { - "id": "2:1100881396300803213", - "name": "frequency", - "type": 5 - }, - { - "id": "3:3482550679443532896", - "name": "isCustom", - "type": 1 - }, - { - "id": "4:7366572918924354162", - "name": "langId", - "type": 5 - }, - { - "id": "5:2610279871343053806", - "name": "length", - "type": 5 - }, - { - "id": "6:5269773217039117329", - "name": "sequence", - "indexId": "1:2971223841434624317", - "type": 9, - "flags": 2048 - }, - { - "id": "7:3922044271904033267", - "name": "sequenceShort", - "indexId": "2:2641086768976362614", - "type": 2, - "flags": 8 - }, - { - "id": "8:1684236207225806285", - "name": "uniqueId", - "indexId": "3:5820769207826940948", - "type": 9, - "flags": 34848 - }, - { - "id": "9:148562041880145406", - "name": "word", - "type": 9 - } - ], - "relations": [] - } - ], - "lastEntityId": "1:8686759156275895115", - "lastIndexId": "3:5820769207826940948", - "lastRelationId": "0:0", - "lastSequenceId": "0:0", - "modelVersion": 5, - "modelVersionParserMinimum": 5, - "retiredEntityUids": [], - "retiredIndexUids": [], - "retiredPropertyUids": [], - "retiredRelationUids": [], - "version": 1 -} \ No newline at end of file diff --git a/res/values/strings.xml b/res/values/strings.xml index d1a0c83c..1185726d 100644 --- a/res/values/strings.xml +++ b/res/values/strings.xml @@ -25,11 +25,11 @@ Compatibility Appearance Debug Options - Recent Log Messages Predictive Mode Select Hotkeys Keypad Initial Setup + Usage Stats Automatic Letter Select Automatically type the selected letter after a short delay. @@ -44,8 +44,6 @@ Yes No Auto - Detailed Debug Logs - System Logs Character for Double 0-key Press Send messages with OK in Messenger Allows sending messages with the OK key in Facebook Messenger. diff --git a/res/xml/prefs_screen_debug.xml b/res/xml/prefs_screen_debug.xml index 4a2dcb7c..a25d1947 100644 --- a/res/xml/prefs_screen_debug.xml +++ b/res/xml/prefs_screen_debug.xml @@ -1,20 +1,26 @@ + + + app:title="Debug Logs" /> + app:title="System Logs" /> + + + + + + + + + + + + + + + diff --git a/scripts/sort-dictionary.js b/scripts/sort-dictionary.js new file mode 100644 index 00000000..297b17c3 --- /dev/null +++ b/scripts/sort-dictionary.js @@ -0,0 +1,123 @@ +const { basename } = require('path'); +const { createReadStream, existsSync } = require('fs'); +const { createInterface } = require('readline'); + + +function printHelp() { + console.log(`Usage ${basename(process.argv[1])} LOCALE WORD-LIST.txt LANGUAGE-DEFINITION.yml`); + console.log('Sorts a dictionary for optimum search speed.'); +} + + + +function validateInput() { + if (process.argv.length < 4) { + printHelp(); + process.exit(1); + } + + if (!existsSync(process.argv[3])) { + console.error(`Failure! Could not find word list file "${process.argv[3]}."`); + process.exit(2); + } + + if (!existsSync(process.argv[4])) { + console.error(`Failure! Could not find language definition file "${process.argv[3]}."`); + process.exit(2); + } + + return { + definitionFile: process.argv[4], + wordsFile: process.argv[3], + locale: process.argv[2] + }; +} + + +function printWords(wordList) { + if (Array.isArray(wordList)) { + wordList.forEach(w => console.log(`${w.word}${w.frequency ? '\t' + w.frequency : ''}`)); + } +} + + +async function readWords(fileName) { + const words = []; + + if (!fileName) { + return words; + } + + for await (const line of createInterface({ input: createReadStream(fileName) })) { + const [word, frequency] = line.split("\t"); + words.push({ + word, + frequency: Number.isNaN(Number.parseInt(frequency)) ? 0 : Number.parseInt(frequency) + }); + } + + return words; +} + + +async function readDefinition(fileName) { + if (!fileName) { + return new Map(); + } + + let lettersPattern = /^\s+-\s*\[([^\]]+)/; + let letterWeights = new Map([["'", 1], ['-', 1], ['"', 1]]); + + let key = 2; + for await (const line of createInterface({ input: createReadStream(fileName) })) { + if (line.includes('SPECIAL') || line.includes('PUNCTUATION')) { + continue; + } + + const matches = line.match(lettersPattern); + if (matches && matches[1]) { + const letters = matches[1].replace(/\s/g, '').split(','); + letters.forEach(l => letterWeights.set(l, key)); + key++; + } + } + + return letterWeights; +} + + +function dictionarySort(a, b, letterWeights, locale) { + if (a.word.length !== b.word.length) { + return a.word.length - b.word.length; + } + + for (let i = 0, end = a.word.length; i < end; i++) { + const charA = a.word.toLocaleLowerCase(locale).charAt(i); + const charB = b.word.toLocaleLowerCase(locale).charAt(i); + const distance = letterWeights.get(charA) - letterWeights.get(charB); + + + if (distance !== 0) { + return distance; + } + } + + return 0; +} + + +async function work({ definitionFile, wordsFile, locale }) { + return Promise.all([ + readWords(wordsFile), + readDefinition(definitionFile) + ]).then(([words, letterWeights]) => + words.sort((a, b) => dictionarySort(a, b, letterWeights, locale)) + ); +} + + + +/** main **/ +work(validateInput()) + .then(words => printWords(words)) + .catch(e => console.error(e)); \ No newline at end of file diff --git a/src/io/github/sspanak/tt9/Logger.java b/src/io/github/sspanak/tt9/Logger.java index c768105f..6a80dc07 100644 --- a/src/io/github/sspanak/tt9/Logger.java +++ b/src/io/github/sspanak/tt9/Logger.java @@ -7,7 +7,7 @@ public class Logger { public static int LEVEL = BuildConfig.DEBUG ? Log.DEBUG : Log.ERROR; public static boolean isDebugLevel() { - return LEVEL == Log.DEBUG; + return LEVEL <= Log.DEBUG; } public static void enableDebugLevel(boolean yes) { @@ -16,7 +16,7 @@ public class Logger { static public void v(String tag, String msg) { if (LEVEL <= Log.VERBOSE) { - Log.v(TAG_PREFIX + tag, msg); + Log.d(TAG_PREFIX + tag, msg); } } diff --git a/src/io/github/sspanak/tt9/TextTools.java b/src/io/github/sspanak/tt9/TextTools.java index 219f135e..0c17fd27 100644 --- a/src/io/github/sspanak/tt9/TextTools.java +++ b/src/io/github/sspanak/tt9/TextTools.java @@ -1,5 +1,9 @@ package io.github.sspanak.tt9; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Locale; +import java.util.TimeZone; import java.util.regex.Pattern; public class TextTools { @@ -37,7 +41,14 @@ public class TextTools { return str != null && !str.isEmpty() && (str.charAt(0) >= '0' && str.charAt(0) <= '9'); } - public static String removeNonLetters(String str) { - return str != null ? str.replaceAll("\\P{L}", "") : null; + public static String unixTimestampToISODate(long timestamp) { + if (timestamp < 0) { + return "--"; + } + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US); + sdf.setTimeZone(TimeZone.getDefault()); + + return sdf.format(new Date(timestamp)); } } diff --git a/src/io/github/sspanak/tt9/db/DictionaryDb.java b/src/io/github/sspanak/tt9/db/DictionaryDb.java deleted file mode 100644 index 2452f013..00000000 --- a/src/io/github/sspanak/tt9/db/DictionaryDb.java +++ /dev/null @@ -1,271 +0,0 @@ -package io.github.sspanak.tt9.db; - -import android.content.Context; -import android.os.Handler; - -import androidx.annotation.NonNull; - -import java.util.ArrayList; -import java.util.List; - -import io.github.sspanak.tt9.ConsumerCompat; -import io.github.sspanak.tt9.Logger; -import io.github.sspanak.tt9.db.exceptions.InsertBlankWordException; -import io.github.sspanak.tt9.db.objectbox.Word; -import io.github.sspanak.tt9.db.objectbox.WordList; -import io.github.sspanak.tt9.db.objectbox.WordStore; -import io.github.sspanak.tt9.ime.TraditionalT9; -import io.github.sspanak.tt9.languages.Language; -import io.github.sspanak.tt9.preferences.SettingsStore; -import io.objectbox.exception.UniqueViolationException; - -public class DictionaryDb { - private static WordStore store; - private static final Handler asyncHandler = new Handler(); - - public static synchronized void init(Context context) { - if (store == null) { - context = context == null ? TraditionalT9.getMainContext() : context; - store = new WordStore(context); - } - } - - - public static synchronized void init() { - init(null); - } - - - private static WordStore getStore() { - init(); - return store; - } - - - private static void printLoadDebug(String sequence, WordList words, long startTime) { - if (!Logger.isDebugLevel()) { - return; - } - - StringBuilder debugText = new StringBuilder("===== Word Matches ====="); - debugText - .append("\n") - .append("Word Count: ").append(words.size()) - .append(". Time: ").append(System.currentTimeMillis() - startTime).append(" ms."); - if (words.size() > 0) { - debugText.append("\n").append(words); - } else { - debugText.append(" Sequence: ").append(sequence); - } - - Logger.d("loadWords", debugText.toString()); - } - - - public static void runInTransaction(Runnable r) { - getStore().runInTransaction(r); - } - - - /** - * normalizeWordFrequencies - * Normalizes the word frequencies for all languages that have reached the maximum, as defined in - * the settings. - * This query will finish immediately, if there is nothing to do. It's safe to run it often. - */ - public static void normalizeWordFrequencies(SettingsStore settings) { - final String LOG_TAG = "db.normalizeWordFrequencies"; - - new Thread(() -> { - for (int langId : getStore().getLanguages()) { - getStore().runInTransactionAsync(() -> { - try { - long start = System.currentTimeMillis(); - - if (getStore().getMaxFrequency(langId) < settings.getWordFrequencyMax()) { - return; - } - - List words = getStore().getMany(langId); - if (words == null) { - return; - } - - for (Word w : words) { - w.frequency /= settings.getWordFrequencyNormalizationDivider(); - } - - getStore().put(words); - - Logger.d( - LOG_TAG, - "Normalized language: " + langId + ", " + words.size() + " words in: " + (System.currentTimeMillis() - start) + " ms" - ); - } catch (Exception e) { - Logger.e(LOG_TAG, "Word normalization failed. " + e.getMessage()); - } finally { - getStore().closeThreadResources(); - } - }); - } - }).start(); - } - - - public static void areThereWords(ConsumerCompat notification, Language language) { - new Thread(() -> { - boolean areThere = getStore().count(language != null ? language.getId() : -1) > 0; - getStore().closeThreadResources(); - notification.accept(areThere); - }).start(); - } - - - public static void deleteWords(Context context, Runnable notification) { - new Thread(() -> { - getStore().destroy(); - store = null; - init(context); - notification.run(); - }).start(); - } - - - public static void deleteWords(Runnable notification, @NonNull ArrayList languageIds) { - new Thread(() -> { - getStore().removeMany(languageIds).closeThreadResources(); - notification.run(); - }).start(); - } - - - public static void insertWord(ConsumerCompat statusHandler, @NonNull Language language, String word) throws Exception { - if (word == null || word.length() == 0) { - throw new InsertBlankWordException(); - } - - new Thread(() -> { - try { - if (getStore().exists(language.getId(), word, language.getDigitSequenceForWord(word))) { - throw new UniqueViolationException("Word already exists"); - } - getStore().put(Word.create(language, word, 1, true)); - statusHandler.accept(0); - } catch (UniqueViolationException e) { - String msg = "Skipping word: '" + word + "' for language: " + language.getId() + ", because it already exists."; - Logger.w("insertWord", msg); - statusHandler.accept(1); - } catch (Exception e) { - String msg = "Failed inserting word: '" + word + "' for language: " + language.getId() + ". " + e.getMessage(); - Logger.e("insertWord", msg); - statusHandler.accept(2); - } finally { - getStore().closeThreadResources(); - } - }).start(); - } - - - public static void upsertWordsSync(List words) { - getStore().put(words); - getStore().closeThreadResources(); - } - - - public static void incrementWordFrequency(@NonNull Language language, @NonNull String word, @NonNull String sequence) { - // If any of these is empty, it is the same as changing the frequency of: "", which is simply a no-op. - if (word.length() == 0 || sequence.length() == 0) { - return; - } - - new Thread(() -> { - try { - long start = System.currentTimeMillis(); - - Word dbWord = getStore().get(language.getId(), word, sequence); - - // In case the user has changed the text case, there would be no match. - // Try again with the lowercase equivalent. - if (dbWord == null) { - dbWord = getStore().get(language.getId(), word.toLowerCase(language.getLocale()), sequence); - } - - if (dbWord == null) { - throw new Exception("No such word"); - } - - int max = getStore().getMaxFrequency(dbWord.langId, dbWord.sequence, dbWord.word); - if (dbWord.frequency <= max) { - dbWord.frequency = max + 1; - getStore().put(dbWord); - long time = System.currentTimeMillis() - start; - - Logger.d( - "incrementWordFrequency", - "Incremented frequency of '" + dbWord.word + "' to: " + dbWord.frequency + ". Time: " + time + " ms" - ); - } else { - long time = System.currentTimeMillis() - start; - Logger.d( - "incrementWordFrequency", - "'" + dbWord.word + "' is already the top word. Keeping frequency: " + dbWord.frequency + ". Time: " + time + " ms" - ); - } - } catch (Exception e) { - Logger.e( - DictionaryDb.class.getName(), - "Failed incrementing word frequency. Word: " + word + ". " + e.getMessage() - ); - } finally { - getStore().closeThreadResources(); - } - }).start(); - } - - - /** - * loadWords - * Loads words matching and similar to a given digit sequence - * For example: "7655" -> "roll" (exact match), but also: "rolled", "roller", "rolling", ... - * and other similar. - */ - private static ArrayList loadWords(Language language, String sequence, String filter, int minimumWords, int maximumWords) { - long start = System.currentTimeMillis(); - - WordList matches = getStore() - .getMany(language, sequence, filter, maximumWords) - .filter(sequence.length(), minimumWords); - - getStore().closeThreadResources(); - printLoadDebug(sequence, matches, start); - return matches.toStringList(); - } - - - public static void getWords(ConsumerCompat> dataHandler, Language language, String sequence, String filter, int minimumWords, int maximumWords) { - final int minWords = Math.max(minimumWords, 0); - final int maxWords = Math.max(maximumWords, minWords); - - if (sequence == null || sequence.length() == 0) { - Logger.w("db.getWords", "Attempting to get words for an empty sequence."); - sendWords(dataHandler, new ArrayList<>()); - return; - } - - if (language == null) { - Logger.w("db.getWords", "Attempting to get words for NULL language."); - sendWords(dataHandler, new ArrayList<>()); - return; - } - - new Thread(() -> sendWords( - dataHandler, - loadWords(language, sequence, filter, minWords, maxWords)) - ).start(); - } - - - private static void sendWords(ConsumerCompat> dataHandler, ArrayList wordList) { - asyncHandler.post(() -> dataHandler.accept(wordList)); - } -} diff --git a/src/io/github/sspanak/tt9/db/DictionaryLoader.java b/src/io/github/sspanak/tt9/db/DictionaryLoader.java index 0e6670c3..4459ead7 100644 --- a/src/io/github/sspanak/tt9/db/DictionaryLoader.java +++ b/src/io/github/sspanak/tt9/db/DictionaryLoader.java @@ -13,21 +13,25 @@ import java.util.Locale; import io.github.sspanak.tt9.ConsumerCompat; import io.github.sspanak.tt9.Logger; +import io.github.sspanak.tt9.db.entities.WordBatch; import io.github.sspanak.tt9.db.exceptions.DictionaryImportAbortedException; import io.github.sspanak.tt9.db.exceptions.DictionaryImportAlreadyRunningException; import io.github.sspanak.tt9.db.exceptions.DictionaryImportException; -import io.github.sspanak.tt9.db.objectbox.Word; +import io.github.sspanak.tt9.db.sqlite.DeleteOps; +import io.github.sspanak.tt9.db.sqlite.InsertOps; +import io.github.sspanak.tt9.db.sqlite.SQLiteOpener; +import io.github.sspanak.tt9.db.sqlite.Tables; import io.github.sspanak.tt9.languages.InvalidLanguageCharactersException; import io.github.sspanak.tt9.languages.InvalidLanguageException; import io.github.sspanak.tt9.languages.Language; import io.github.sspanak.tt9.preferences.SettingsStore; public class DictionaryLoader { + private static final String LOG_TAG = "DictionaryLoader"; private static DictionaryLoader self; - private final String logTag = "DictionaryLoader"; private final AssetManager assets; - private final SettingsStore settings; + private final SQLiteOpener sqlite; private static final Handler asyncHandler = new Handler(); private ConsumerCompat onStatusChange; @@ -50,7 +54,7 @@ public class DictionaryLoader { public DictionaryLoader(Context context) { assets = context.getAssets(); - settings = new SettingsStore(context); + sqlite = SQLiteOpener.getInstance(context); } @@ -70,7 +74,7 @@ public class DictionaryLoader { } if (languages.size() == 0) { - Logger.d(logTag, "Nothing to do"); + Logger.d(LOG_TAG, "Nothing to do"); return; } @@ -109,118 +113,197 @@ public class DictionaryLoader { private void importAll(Language language) { if (language == null) { - Logger.e(logTag, "Failed loading a dictionary for NULL language."); + Logger.e(LOG_TAG, "Failed loading a dictionary for NULL language."); sendError(InvalidLanguageException.class.getSimpleName(), -1); return; } - DictionaryDb.runInTransaction(() -> { - try { - long start = System.currentTimeMillis(); - importWords(language, language.getDictionaryFile()); - Logger.i( - logTag, - "Dictionary: '" + language.getDictionaryFile() + "'" + - " processing time: " + (System.currentTimeMillis() - start) + " ms" - ); + try { + long start = System.currentTimeMillis(); + float progress = 1; + final float dictionaryMaxProgress = 90f; - start = System.currentTimeMillis(); - importLetters(language); - Logger.i( - logTag, - "Loaded letters for '" + language.getName() + "' language in: " + (System.currentTimeMillis() - start) + " ms" - ); - } catch (DictionaryImportAbortedException e) { - stop(); + sqlite.beginTransaction(); - Logger.i( - logTag, - e.getMessage() + ". File '" + language.getDictionaryFile() + "' not imported." - ); - } catch (DictionaryImportException e) { - stop(); - sendImportError(DictionaryImportException.class.getSimpleName(), language.getId(), e.line, e.word); + Tables.dropIndexes(sqlite.getDb(), language); + sendProgressMessage(language, ++progress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME); + logLoadingStep("Indexes dropped", language, start); - Logger.e( - logTag, - " Invalid word: '" + e.word - + "' in dictionary: '" + language.getDictionaryFile() + "'" - + " on line " + e.line - + " of language '" + language.getName() + "'. " - + e.getMessage() - ); - } catch (Exception | Error e) { - stop(); - sendError(e.getClass().getSimpleName(), language.getId()); + start = System.currentTimeMillis(); + DeleteOps.delete(sqlite, language.getId()); + sendProgressMessage(language, ++progress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME); + logLoadingStep("Storage cleared", language, start); - Logger.e( - logTag, - "Failed loading dictionary: " + language.getDictionaryFile() - + " for language '" + language.getName() + "'. " - + e.getMessage() - ); - } - }); + start = System.currentTimeMillis(); + int lettersCount = importLetters(language); + sendProgressMessage(language, ++progress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME); + logLoadingStep("Letters imported", language, start); + + start = System.currentTimeMillis(); + InsertOps.restoreCustomWords(sqlite.getDb(), language); + sendProgressMessage(language, ++progress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME); + logLoadingStep("Custom words restored", language, start); + + start = System.currentTimeMillis(); + WordBatch words = readWordsFile(language, lettersCount, progress, progress + 25f); + progress += 25; + sendProgressMessage(language, progress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME); + logLoadingStep("Dictionary file loaded in memory", language, start); + + start = System.currentTimeMillis(); + saveWordBatch(words, progress, dictionaryMaxProgress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME); + progress = dictionaryMaxProgress; + sendProgressMessage(language, progress, 0); + logLoadingStep("Dictionary words saved in database", language, start); + + start = System.currentTimeMillis(); + Tables.createPositionIndex(sqlite.getDb(), language); + sendProgressMessage(language, progress + (100f - progress) / 2f, 0); + Tables.createWordIndex(sqlite.getDb(), language); + sendProgressMessage(language, 100, 0); + logLoadingStep("Indexes restored", language, start); + + sqlite.finishTransaction(); + SlowQueryStats.clear(); + } catch (DictionaryImportAbortedException e) { + sqlite.failTransaction(); + stop(); + Logger.i(LOG_TAG, e.getMessage() + ". File '" + language.getDictionaryFile() + "' not imported."); + } catch (DictionaryImportException e) { + stop(); + sqlite.failTransaction(); + sendImportError(DictionaryImportException.class.getSimpleName(), language.getId(), e.line, e.word); + + Logger.e( + LOG_TAG, + " Invalid word: '" + e.word + + "' in dictionary: '" + language.getDictionaryFile() + "'" + + " on line " + e.line + + " of language '" + language.getName() + "'. " + + e.getMessage() + ); + } catch (Exception | Error e) { + stop(); + sqlite.failTransaction(); + sendError(e.getClass().getSimpleName(), language.getId()); + + Logger.e( + LOG_TAG, + "Failed loading dictionary: " + language.getDictionaryFile() + + " for language '" + language.getName() + "'. " + + e.getMessage() + ); + } } - private void importLetters(Language language) throws InvalidLanguageCharactersException { - ArrayList letters = new ArrayList<>(); - + private int importLetters(Language language) throws InvalidLanguageCharactersException, DictionaryImportAbortedException { + int lettersCount = 0; boolean isEnglish = language.getLocale().equals(Locale.ENGLISH); + WordBatch letters = new WordBatch(language); for (int key = 2; key <= 9; key++) { for (String langChar : language.getKeyCharacters(key, false)) { langChar = (isEnglish && langChar.equals("i")) ? langChar.toUpperCase(Locale.ENGLISH) : langChar; - letters.add(Word.create(language, langChar, 0)); + letters.add(langChar, 0, key); + lettersCount++; } } - DictionaryDb.upsertWordsSync(letters); + saveWordBatch(letters, -1, -1, -1); + + return lettersCount; } - private void importWords(Language language, String dictionaryFile) throws Exception { - sendProgressMessage(language, 1, 0); + private WordBatch readWordsFile(Language language, int positionShift, float minProgress, float maxProgress) throws Exception { + int currentLine = 1; + int totalLines = getFileSize(language.getDictionaryFile()); + float progressRatio = (maxProgress - minProgress) / totalLines; - long currentLine = 0; - long totalLines = getFileSize(dictionaryFile); + WordBatch batch = new WordBatch(language, totalLines); - BufferedReader br = new BufferedReader(new InputStreamReader(assets.open(dictionaryFile), StandardCharsets.UTF_8)); - ArrayList dbWords = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new InputStreamReader(assets.open(language.getDictionaryFile()), StandardCharsets.UTF_8))) { + for (String line; (line = br.readLine()) != null; currentLine++) { + if (loadThread.isInterrupted()) { + sendProgressMessage(language, 0, 0); + throw new DictionaryImportAbortedException(); + } - for (String line; (line = br.readLine()) != null; currentLine++) { - if (loadThread.isInterrupted()) { - br.close(); - sendProgressMessage(language, 0, 0); - throw new DictionaryImportAbortedException(); - } + String[] parts = splitLine(line); + String word = parts[0]; + short frequency = getFrequency(parts); - String[] parts = splitLine(line); - String word = parts[0]; - int frequency = getFrequency(parts); + try { + batch.add(word, frequency, currentLine + positionShift); + } catch (InvalidLanguageCharactersException e) { + throw new DictionaryImportException(word, currentLine); + } - try { - dbWords.add(Word.create(language, word, frequency)); - } catch (InvalidLanguageCharactersException e) { - br.close(); - throw new DictionaryImportException(word, currentLine); - } - - if (dbWords.size() >= settings.getDictionaryImportWordChunkSize() || currentLine >= totalLines - 1) { - DictionaryDb.upsertWordsSync(dbWords); - dbWords.clear(); - } - - if (totalLines > 0) { - int progress = (int) Math.floor(100.0 * currentLine / totalLines); - progress = Math.max(1, progress); - sendProgressMessage(language, progress, settings.getDictionaryImportProgressUpdateInterval()); + if (totalLines > 0 && currentLine % SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_BATCH_SIZE == 0) { + sendProgressMessage(language, minProgress + progressRatio * currentLine, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME); + } } } - br.close(); - sendProgressMessage(language, 100, 0); + return batch; + } + + + public void saveWordBatch(WordBatch batch, float minProgress, float maxProgress, int sizeUpdateInterval) throws DictionaryImportAbortedException { + float middleProgress = minProgress + (maxProgress - minProgress) / 2; + + InsertOps insertOps = new InsertOps(sqlite.getDb(), batch.getLanguage()); + + insertWordsBatch(insertOps, batch, minProgress, middleProgress - 2, sizeUpdateInterval); + insertWordPositionsBatch(insertOps, batch, middleProgress - 2, maxProgress - 2, sizeUpdateInterval); + InsertOps.insertLanguageMeta(sqlite.getDb(), batch.getLanguage().getId()); + + if (sizeUpdateInterval > 0) { + sendProgressMessage(batch.getLanguage(), maxProgress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME); + } + } + + + private void insertWordsBatch(InsertOps insertOps, WordBatch batch, float minProgress, float maxProgress, int sizeUpdateInterval) throws DictionaryImportAbortedException { + if (batch.getWords().size() == 0) { + return; + } + + float progressRatio = (maxProgress - minProgress) / batch.getWords().size(); + + for (int progress = 0, end = batch.getWords().size(); progress < end; progress++) { + if (loadThread.isInterrupted()) { + sendProgressMessage(batch.getLanguage(), 0, 0); + throw new DictionaryImportAbortedException(); + } + + insertOps.insertWord(batch.getWords().get(progress)); + if (sizeUpdateInterval > 0 && progress % sizeUpdateInterval == 0) { + sendProgressMessage(batch.getLanguage(), minProgress + progress * progressRatio, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME); + } + } + } + + + private void insertWordPositionsBatch(InsertOps insertOps, WordBatch batch, float minProgress, float maxProgress, int sizeUpdateInterval) throws DictionaryImportAbortedException { + if (batch.getPositions().size() == 0) { + return; + } + + float progressRatio = (maxProgress - minProgress) / batch.getPositions().size(); + + for (int progress = 0, end = batch.getPositions().size(); progress < end; progress++) { + if (loadThread.isInterrupted()) { + sendProgressMessage(batch.getLanguage(), 0, 0); + throw new DictionaryImportAbortedException(); + } + + insertOps.insertWordPosition(batch.getPositions().get(progress)); + if (sizeUpdateInterval > 0 && progress % sizeUpdateInterval == 0) { + sendProgressMessage(batch.getLanguage(), minProgress + progress * progressRatio, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME); + } + } } @@ -241,21 +324,21 @@ public class DictionaryLoader { } - private long getFileSize(String filename) { + private int getFileSize(String filename) { String sizeFilename = filename + ".size"; try (BufferedReader reader = new BufferedReader(new InputStreamReader(assets.open(sizeFilename), StandardCharsets.UTF_8))) { return Integer.parseInt(reader.readLine()); } catch (Exception e) { - Logger.w(logTag, "Could not read the size of: " + filename + " from: " + sizeFilename + ". " + e.getMessage()); + Logger.w(LOG_TAG, "Could not read the size of: " + filename + " from: " + sizeFilename + ". " + e.getMessage()); return 0; } } - private int getFrequency(String[] lineParts) { + private short getFrequency(String[] lineParts) { try { - return Integer.parseInt(lineParts[1]); + return Short.parseShort(lineParts[1]); } catch (Exception e) { return 0; } @@ -264,7 +347,7 @@ public class DictionaryLoader { private void sendStartMessage(int fileCount) { if (onStatusChange == null) { - Logger.w(logTag, "Cannot send file count without a status Handler. Ignoring message."); + Logger.w(LOG_TAG, "Cannot send file count without a status Handler. Ignoring message."); return; } @@ -275,9 +358,9 @@ public class DictionaryLoader { } - private void sendProgressMessage(Language language, int progress, int progressUpdateInterval) { + private void sendProgressMessage(Language language, float progress, int progressUpdateInterval) { if (onStatusChange == null) { - Logger.w(logTag, "Cannot send progress without a status Handler. Ignoring message."); + Logger.w(LOG_TAG, "Cannot send progress without a status Handler. Ignoring message."); return; } @@ -291,7 +374,7 @@ public class DictionaryLoader { Bundle progressMsg = new Bundle(); progressMsg.putInt("languageId", language.getId()); progressMsg.putLong("time", getImportTime()); - progressMsg.putInt("progress", progress); + progressMsg.putInt("progress", (int) Math.round(progress)); progressMsg.putInt("currentFile", currentFile); asyncHandler.post(() -> onStatusChange.accept(progressMsg)); } @@ -299,7 +382,7 @@ public class DictionaryLoader { private void sendError(String message, int langId) { if (onStatusChange == null) { - Logger.w(logTag, "Cannot send an error without a status Handler. Ignoring message."); + Logger.w(LOG_TAG, "Cannot send an error without a status Handler. Ignoring message."); return; } @@ -312,7 +395,7 @@ public class DictionaryLoader { private void sendImportError(String message, int langId, long fileLine, String word) { if (onStatusChange == null) { - Logger.w(logTag, "Cannot send an import error without a status Handler. Ignoring message."); + Logger.w(LOG_TAG, "Cannot send an import error without a status Handler. Ignoring message."); return; } @@ -323,4 +406,11 @@ public class DictionaryLoader { errorMsg.putString("word", word); asyncHandler.post(() -> onStatusChange.accept(errorMsg)); } + + + private void logLoadingStep(String message, Language language, long time) { + if (Logger.isDebugLevel()) { + Logger.d(LOG_TAG, message + " for language '" + language.getName() + "' in: " + (System.currentTimeMillis() - time) + " ms"); + } + } } diff --git a/src/io/github/sspanak/tt9/db/LegacyDb.java b/src/io/github/sspanak/tt9/db/LegacyDb.java new file mode 100644 index 00000000..bb0ddd6a --- /dev/null +++ b/src/io/github/sspanak/tt9/db/LegacyDb.java @@ -0,0 +1,39 @@ +package io.github.sspanak.tt9.db; + +import android.app.Activity; +import android.database.sqlite.SQLiteDatabase; +import android.database.sqlite.SQLiteOpenHelper; + +import io.github.sspanak.tt9.Logger; + +public class LegacyDb extends SQLiteOpenHelper { + private final String LOG_TAG = getClass().getSimpleName(); + private static final String DB_NAME = "t9dict.db"; + private static final String TABLE_NAME = "words"; + + private static boolean isCompleted = false; + + public LegacyDb(Activity activity) { + super(activity.getApplicationContext(), DB_NAME, null, 12); + } + + public void clear() { + if (isCompleted) { + return; + } + + new Thread(() -> { + try (SQLiteDatabase db = getWritableDatabase()) { + db.compileStatement("DROP TABLE " + TABLE_NAME).execute(); + Logger.d(LOG_TAG, "SQL Words cleaned successfully."); + } catch (Exception e) { + Logger.d(LOG_TAG, "Assuming no words, because of query error. " + e.getMessage()); + } finally { + isCompleted = true; + } + }).start(); + } + + @Override public void onCreate(SQLiteDatabase db) {} + @Override public void onUpgrade(SQLiteDatabase db, int i, int ii) {} +} diff --git a/src/io/github/sspanak/tt9/db/SQLWords.java b/src/io/github/sspanak/tt9/db/SQLWords.java deleted file mode 100644 index 28f5192c..00000000 --- a/src/io/github/sspanak/tt9/db/SQLWords.java +++ /dev/null @@ -1,78 +0,0 @@ -package io.github.sspanak.tt9.db; - -import android.app.Activity; -import android.database.Cursor; -import android.database.sqlite.SQLiteDatabase; - -import java.io.File; - -import io.github.sspanak.tt9.Logger; - -public class SQLWords { - private static final String DB_NAME = "t9dict.db"; - private static final String TABLE_NAME = "words"; - - private static boolean isCompleted = false; - - private final String LOG_TAG; - private final Activity activity; - private SQLiteDatabase db; - - public SQLWords(Activity activity) { - this.activity = activity; - - LOG_TAG = getClass().getSimpleName(); - } - - public void clear() { - if (isCompleted) { - return; - } - - new Thread(() -> { - openDb(); - if (areThereWords()) { - deleteAll(); - } - closeDb(); - isCompleted = true; - }).start(); - } - - - private void openDb() { - try { - db = null; - File dbFile = activity.getDatabasePath(DB_NAME); - if (dbFile.exists()) { - db = SQLiteDatabase.openDatabase(dbFile.getAbsolutePath(), null, SQLiteDatabase.OPEN_READWRITE); - } - } catch (Exception e) { - Logger.d(LOG_TAG, "Assuming no SQL database, because of error while opening. " + e.getMessage()); - db = null; - } - } - - - private void closeDb() { - if (db != null) { - db.close(); - } - } - - - private boolean areThereWords() { - String sql = "SELECT COUNT(*) FROM (SELECT id FROM " + TABLE_NAME + " LIMIT 1)"; - try (Cursor cursor = db.rawQuery(sql, null)) { - return cursor.moveToFirst() && cursor.getInt(0) > 0; - } catch (Exception e) { - Logger.d(LOG_TAG, "Assuming no words, because of query error. " + e.getMessage()); - return false; - } - } - - private void deleteAll() { - db.execSQL("DROP TABLE words"); - Logger.d(LOG_TAG, "SQL Words cleaned successfully."); - } -} diff --git a/src/io/github/sspanak/tt9/db/SlowQueryStats.java b/src/io/github/sspanak/tt9/db/SlowQueryStats.java new file mode 100644 index 00000000..826c8620 --- /dev/null +++ b/src/io/github/sspanak/tt9/db/SlowQueryStats.java @@ -0,0 +1,87 @@ +package io.github.sspanak.tt9.db; + +import java.util.HashMap; + +import io.github.sspanak.tt9.Logger; +import io.github.sspanak.tt9.TextTools; +import io.github.sspanak.tt9.languages.Language; +import io.github.sspanak.tt9.preferences.SettingsStore; + +public class SlowQueryStats { + private static final String LOG_TAG = SlowQueryStats.class.getSimpleName(); + private static long firstQueryTime = -1; + private static long maxQueryTime = 0; + private static long totalQueries = 0; + private static long totalQueryTime = 0; + private static final HashMap slowQueries = new HashMap<>(); + private static final HashMap resultCache = new HashMap<>(); + + + public static String generateKey(Language language, String sequence, String wordFilter, int minimumWords) { + return language.getId() + "_" + sequence + "_" + wordFilter + "_" + minimumWords; + } + + public static void add(String key, int time, String positionsList) { + if (firstQueryTime == -1) { + firstQueryTime = System.currentTimeMillis(); + } + maxQueryTime = Math.max(maxQueryTime, time); + totalQueries++; + totalQueryTime += time; + if (time < SettingsStore.SLOW_QUERY_TIME) { + return; + } + + slowQueries.put(key, time); + if (!resultCache.containsKey(key)) { + resultCache.put(key, positionsList.replaceAll("-\\d+,", "")); + } + } + + public static String getCachedIfSlow(String key) { + Integer queryTime = slowQueries.get(key); + boolean isSlow = queryTime != null && queryTime >= SettingsStore.SLOW_QUERY_TIME; + + if (isSlow) { + Logger.d(LOG_TAG, "Loading cached positions for query: " + key); + return resultCache.get(key); + } else { + return null; + } + } + + public static String getSummary() { + long slowQueryTotalTime = 0; + for (int time : slowQueries.values()) { + slowQueryTotalTime += time; + } + + long averageTime = totalQueries == 0 ? 0 : totalQueryTime / totalQueries; + long slowAverageTime = slowQueries.size() == 0 ? 0 : slowQueryTotalTime / slowQueries.size(); + + return + "Queries: " + totalQueries + ". Average time: " + averageTime + " ms." + + "\nSlow: " + slowQueries.size() + ". Average time: " + slowAverageTime + " ms." + + "\nSlowest: " + maxQueryTime + " ms." + + "\nFirst: " + TextTools.unixTimestampToISODate(firstQueryTime); + } + + public static String getList() { + StringBuilder sb = new StringBuilder(); + for (String key : slowQueries.keySet()) { + sb.append(key).append(": ").append(slowQueries.get(key)).append(" ms\n"); + } + + return sb.toString(); + } + + + public static void clear() { + firstQueryTime = -1; + maxQueryTime = 0; + totalQueries = 0; + totalQueryTime = 0; + slowQueries.clear(); + resultCache.clear(); + } +} diff --git a/src/io/github/sspanak/tt9/db/WordStore.java b/src/io/github/sspanak/tt9/db/WordStore.java new file mode 100644 index 00000000..917729e5 --- /dev/null +++ b/src/io/github/sspanak/tt9/db/WordStore.java @@ -0,0 +1,259 @@ +package io.github.sspanak.tt9.db; + +import android.content.Context; + +import androidx.annotation.NonNull; + +import java.util.ArrayList; + +import io.github.sspanak.tt9.Logger; +import io.github.sspanak.tt9.db.entities.Word; +import io.github.sspanak.tt9.db.entities.WordList; +import io.github.sspanak.tt9.db.sqlite.DeleteOps; +import io.github.sspanak.tt9.db.sqlite.InsertOps; +import io.github.sspanak.tt9.db.sqlite.ReadOps; +import io.github.sspanak.tt9.db.sqlite.SQLiteOpener; +import io.github.sspanak.tt9.db.sqlite.UpdateOps; +import io.github.sspanak.tt9.ime.TraditionalT9; +import io.github.sspanak.tt9.languages.Language; +import io.github.sspanak.tt9.preferences.SettingsStore; +import io.github.sspanak.tt9.ui.AddWordAct; + + +public class WordStore { + private final String LOG_TAG = "sqlite.WordStore"; + private static WordStore self; + + private SQLiteOpener sqlite = null; + private ReadOps readOps = null; + + + public WordStore(@NonNull Context context) { + try { + sqlite = SQLiteOpener.getInstance(context); + sqlite.getDb(); + readOps = new ReadOps(); + } catch (Exception e) { + Logger.w(LOG_TAG, "Database connection failure. All operations will return empty results. " + e.getMessage()); + } + self = this; + } + + + public static synchronized WordStore getInstance(Context context) { + if (self == null) { + context = context == null ? TraditionalT9.getMainContext() : context; + self = new WordStore(context); + } + + return self; + } + + + /** + * Loads words matching and similar to a given digit sequence + * For example: "7655" -> "roll" (exact match), but also: "rolled", "roller", "rolling", ... + * and other similar. + */ + public ArrayList getSimilar(Language language, String sequence, String wordFilter, int minimumWords, int maximumWords) { + if (!checkOrNotify()) { + return new ArrayList<>(); + } + + if (sequence == null || sequence.length() == 0) { + Logger.w(LOG_TAG, "Attempting to get words for an empty sequence."); + return new ArrayList<>(); + } + + if (language == null) { + Logger.w(LOG_TAG, "Attempting to get words for NULL language."); + return new ArrayList<>(); + } + + final int minWords = Math.max(minimumWords, 0); + final int maxWords = Math.max(maximumWords, minWords); + final String filter = wordFilter == null ? "" : wordFilter; + + long startTime = System.currentTimeMillis(); + String positions = readOps.getSimilarWordPositions(sqlite.getDb(), language, sequence, filter, minWords); + long positionsTime = System.currentTimeMillis() - startTime; + + startTime = System.currentTimeMillis(); + ArrayList words = readOps.getWords(sqlite.getDb(), language, positions, filter, maxWords, false).toStringList(); + long wordsTime = System.currentTimeMillis() - startTime; + + printLoadingSummary(sequence, words, positionsTime, wordsTime); + SlowQueryStats.add(SlowQueryStats.generateKey(language, sequence, wordFilter, minWords), (int) (positionsTime + wordsTime), positions); + + return words; + } + + + public boolean exists(Language language) { + return language != null && checkOrNotify() && readOps.exists(sqlite.getDb(), language.getId()); + } + + + public void remove(ArrayList languageIds) { + if (!checkOrNotify()) { + return; + } + + long start = System.currentTimeMillis(); + try { + sqlite.beginTransaction(); + for (int langId : languageIds) { + if (readOps.exists(sqlite.getDb(), langId)) { + DeleteOps.delete(sqlite, langId); + } + } + sqlite.finishTransaction(); + + Logger.d(LOG_TAG, "Deleted " + languageIds.size() + " languages. Time: " + (System.currentTimeMillis() - start) + " ms"); + } catch (Exception e) { + sqlite.failTransaction(); + Logger.e(LOG_TAG, "Failed deleting languages. " + e.getMessage()); + } + } + + + public int put(Language language, String word) { + if (word == null || word.isEmpty()) { + return AddWordAct.CODE_BLANK_WORD; + } + + if (language == null) { + return AddWordAct.CODE_INVALID_LANGUAGE; + } + + if (!checkOrNotify()) { + return AddWordAct.CODE_GENERAL_ERROR; + } + + try { + if (readOps.exists(sqlite.getDb(), language, word)) { + return AddWordAct.CODE_WORD_EXISTS; + } + + String sequence = language.getDigitSequenceForWord(word); + + if (InsertOps.insertCustomWord(sqlite.getDb(), language, sequence, word)) { + makeTopWord(language, word, sequence); + } else { + throw new Exception("SQLite INSERT failure."); + } + } catch (Exception e) { + String msg = "Failed inserting word: '" + word + "' for language: " + language.getId() + ". " + e.getMessage(); + Logger.e("insertWord", msg); + return AddWordAct.CODE_GENERAL_ERROR; + } + + return AddWordAct.CODE_SUCCESS; + } + + + private boolean checkOrNotify() { + if (sqlite == null || sqlite.getDb() == null) { + Logger.e(LOG_TAG, "No database connection. Cannot query any data."); + return false; + } + + return true; + } + + + public void makeTopWord(@NonNull Language language, @NonNull String word, @NonNull String sequence) { + if (!checkOrNotify() || word.isEmpty() || sequence.isEmpty()) { + return; + } + + try { + long start = System.currentTimeMillis(); + + String topWordPositions = readOps.getWordPositions(sqlite.getDb(), language, sequence, 0, 0, ""); + WordList topWords = readOps.getWords(sqlite.getDb(), language, topWordPositions, "", 9999, true); + if (topWords.isEmpty()) { + throw new Exception("No such word"); + } + + Word topWord = topWords.get(0); + if (topWord.word.toUpperCase(language.getLocale()).equals(word.toUpperCase(language.getLocale()))) { + Logger.d(LOG_TAG, "Word '" + word + "' is already the top word. Time: " + (System.currentTimeMillis() - start) + " ms"); + return; + } + + int wordPosition = 0; + for (Word tw : topWords) { + if (tw.word.toUpperCase(language.getLocale()).equals(word.toUpperCase(language.getLocale()))) { + wordPosition = tw.position; + break; + } + } + + int newTopFrequency = topWord.frequency + 1; + String wordFilter = word.length() == 1 ? word.toLowerCase(language.getLocale()) : null; + if (!UpdateOps.changeFrequency(sqlite.getDb(), language, wordFilter, wordPosition, newTopFrequency)) { + throw new Exception("No such word"); + } + + if (newTopFrequency > SettingsStore.WORD_FREQUENCY_MAX) { + scheduleNormalization(language); + } + + Logger.d(LOG_TAG, "Changed frequency of '" + word + "' to: " + newTopFrequency + ". Time: " + (System.currentTimeMillis() - start) + " ms"); + } catch (Exception e) { + Logger.e(LOG_TAG,"Frequency change failed. Word: '" + word + "'. " + e.getMessage()); + } + } + + + public void normalizeNext() { + if (!checkOrNotify()) { + return; + } + + long start = System.currentTimeMillis(); + + try { + sqlite.beginTransaction(); + int nextLangId = readOps.getNextInNormalizationQueue(sqlite.getDb()); + UpdateOps.normalize(sqlite.getDb(), nextLangId); + sqlite.finishTransaction(); + + String message = nextLangId > 0 ? "Normalized language: " + nextLangId : "No languages to normalize"; + Logger.d(LOG_TAG, message + ". Time: " + (System.currentTimeMillis() - start) + " ms"); + } catch (Exception e) { + sqlite.failTransaction(); + Logger.e(LOG_TAG, "Normalization failed. " + e.getMessage()); + } + } + + + public void scheduleNormalization(Language language) { + if (language != null && checkOrNotify()) { + UpdateOps.scheduleNormalization(sqlite.getDb(), language); + } + } + + + private void printLoadingSummary(String sequence, ArrayList words, long positionIndexTime, long wordsTime) { + if (!Logger.isDebugLevel()) { + return; + } + + StringBuilder debugText = new StringBuilder("===== Word Loading Summary ====="); + debugText + .append("\nWord Count: ").append(words.size()) + .append(".\nTime: ").append(positionIndexTime + wordsTime) + .append(" ms (positions: ").append(positionIndexTime) + .append(" ms, words: ").append(wordsTime).append(" ms)."); + + if (words.isEmpty()) { + debugText.append(" Sequence: ").append(sequence); + } else { + debugText.append("\n").append(words); + } + + Logger.d(LOG_TAG, debugText.toString()); + } +} diff --git a/src/io/github/sspanak/tt9/db/WordStoreAsync.java b/src/io/github/sspanak/tt9/db/WordStoreAsync.java new file mode 100644 index 00000000..f36d23c8 --- /dev/null +++ b/src/io/github/sspanak/tt9/db/WordStoreAsync.java @@ -0,0 +1,66 @@ +package io.github.sspanak.tt9.db; + +import android.content.Context; +import android.os.Handler; + +import androidx.annotation.NonNull; + +import java.util.ArrayList; + +import io.github.sspanak.tt9.ConsumerCompat; +import io.github.sspanak.tt9.languages.Language; + +public class WordStoreAsync { + private static WordStore store; + private static final Handler asyncHandler = new Handler(); + + public static synchronized void init(Context context) { + store = WordStore.getInstance(context); + } + + + public static synchronized void init() { + init(null); + } + + + private static WordStore getStore() { + init(); + return store; + } + + + public static void normalizeNext() { + new Thread(() -> getStore().normalizeNext()).start(); + } + + + public static void areThereWords(ConsumerCompat notification, Language language) { + new Thread(() -> notification.accept(getStore().exists(language))).start(); + } + + + public static void deleteWords(Runnable notification, @NonNull ArrayList languageIds) { + new Thread(() -> { + getStore().remove(languageIds); + notification.run(); + }).start(); + } + + + public static void put(ConsumerCompat statusHandler, Language language, String word) { + new Thread(() -> statusHandler.accept(getStore().put(language, word))).start(); + } + + + public static void makeTopWord(@NonNull Language language, @NonNull String word, @NonNull String sequence) { + new Thread(() -> getStore().makeTopWord(language, word, sequence)).start(); + } + + + public static void getWords(ConsumerCompat> dataHandler, Language language, String sequence, String filter, int minWords, int maxWords) { + new Thread(() -> asyncHandler.post(() -> dataHandler.accept( + getStore().getSimilar(language, sequence, filter, minWords, maxWords))) + ).start(); + } +} diff --git a/src/io/github/sspanak/tt9/db/entities/Word.java b/src/io/github/sspanak/tt9/db/entities/Word.java new file mode 100644 index 00000000..a7343ee3 --- /dev/null +++ b/src/io/github/sspanak/tt9/db/entities/Word.java @@ -0,0 +1,18 @@ +package io.github.sspanak.tt9.db.entities; + +import androidx.annotation.NonNull; + +public class Word { + public int frequency; + public int position; + public String word; + + public static Word create(@NonNull String word, int frequency, int position) { + Word w = new Word(); + w.frequency = frequency; + w.position = position; + w.word = word; + + return w; + } +} diff --git a/src/io/github/sspanak/tt9/db/entities/WordBatch.java b/src/io/github/sspanak/tt9/db/entities/WordBatch.java new file mode 100644 index 00000000..574efc93 --- /dev/null +++ b/src/io/github/sspanak/tt9/db/entities/WordBatch.java @@ -0,0 +1,61 @@ +package io.github.sspanak.tt9.db.entities; + +import androidx.annotation.NonNull; + +import java.util.ArrayList; + +import io.github.sspanak.tt9.languages.InvalidLanguageCharactersException; +import io.github.sspanak.tt9.languages.Language; + +public class WordBatch { + @NonNull private final Language language; + @NonNull private final ArrayList words; + @NonNull private final ArrayList positions; + + private WordPosition lastWordPosition; + + public WordBatch(@NonNull Language language, int size) { + this.language = language; + words = size > 0 ? new ArrayList<>(size) : new ArrayList<>(); + positions = size > 0 ? new ArrayList<>(size) : new ArrayList<>(); + } + + public WordBatch(@NonNull Language language) { + this(language, 0); + } + + public void add(@NonNull String word, int frequency, int position) throws InvalidLanguageCharactersException { + words.add(Word.create(word, frequency, position)); + + if (position == 0) { + return; + } + + String sequence = language.getDigitSequenceForWord(word); + + if (position == 1 || lastWordPosition == null) { + lastWordPosition = WordPosition.create(sequence, position); + } else { + lastWordPosition.end = position; + } + + if (!sequence.equals(lastWordPosition.sequence)) { + lastWordPosition.end--; + positions.add(lastWordPosition); + + lastWordPosition = WordPosition.create(sequence, position); + } + } + + @NonNull public Language getLanguage() { + return language; + } + + @NonNull public ArrayList getWords() { + return words; + } + + @NonNull public ArrayList getPositions() { + return positions; + } +} diff --git a/src/io/github/sspanak/tt9/db/entities/WordList.java b/src/io/github/sspanak/tt9/db/entities/WordList.java new file mode 100644 index 00000000..f2728550 --- /dev/null +++ b/src/io/github/sspanak/tt9/db/entities/WordList.java @@ -0,0 +1,17 @@ +package io.github.sspanak.tt9.db.entities; + +import java.util.ArrayList; + +public class WordList extends ArrayList { + public void add(String word, int frequency, int position) { + add(Word.create(word, frequency, position)); + } + + public ArrayList toStringList() { + ArrayList list = new ArrayList<>(size()); + for (Word word : this) { + list.add(word.word); + } + return list; + } +} diff --git a/src/io/github/sspanak/tt9/db/entities/WordPosition.java b/src/io/github/sspanak/tt9/db/entities/WordPosition.java new file mode 100644 index 00000000..0f37ed27 --- /dev/null +++ b/src/io/github/sspanak/tt9/db/entities/WordPosition.java @@ -0,0 +1,17 @@ +package io.github.sspanak.tt9.db.entities; + +import androidx.annotation.NonNull; + +public class WordPosition { + public String sequence; + public int start; + public int end; + + public static WordPosition create(@NonNull String sequence, int start) { + WordPosition position = new WordPosition(); + position.sequence = sequence; + position.start = start; + + return position; + } +} diff --git a/src/io/github/sspanak/tt9/db/entities/WordPositionsStringBuilder.java b/src/io/github/sspanak/tt9/db/entities/WordPositionsStringBuilder.java new file mode 100644 index 00000000..178ba3f3 --- /dev/null +++ b/src/io/github/sspanak/tt9/db/entities/WordPositionsStringBuilder.java @@ -0,0 +1,38 @@ +package io.github.sspanak.tt9.db.entities; + +import android.database.Cursor; + +import androidx.annotation.NonNull; + + +public class WordPositionsStringBuilder { + public int size = 0; + private final StringBuilder positions = new StringBuilder(); + + public WordPositionsStringBuilder appendFromDbRanges(Cursor cursor) { + while (cursor.moveToNext()) { + append(cursor.getInt(0), cursor.getInt(1)); + } + + return this; + } + + private void append(int start, int end) { + if (size > 0) { + positions.append(","); + } + positions.append(start); + size++; + + for (int position = start + 1; position <= end; position++) { + positions.append(",").append(position); + size++; + } + } + + @NonNull + @Override + public String toString() { + return positions.toString(); + } +} diff --git a/src/io/github/sspanak/tt9/db/exceptions/InsertBlankWordException.java b/src/io/github/sspanak/tt9/db/exceptions/InsertBlankWordException.java deleted file mode 100644 index 48eecd84..00000000 --- a/src/io/github/sspanak/tt9/db/exceptions/InsertBlankWordException.java +++ /dev/null @@ -1,7 +0,0 @@ -package io.github.sspanak.tt9.db.exceptions; - -public class InsertBlankWordException extends Exception { - public InsertBlankWordException() { - super("Cannot insert a blank word."); - } -} diff --git a/src/io/github/sspanak/tt9/db/objectbox/Word.java b/src/io/github/sspanak/tt9/db/objectbox/Word.java deleted file mode 100644 index 4eae3de1..00000000 --- a/src/io/github/sspanak/tt9/db/objectbox/Word.java +++ /dev/null @@ -1,48 +0,0 @@ -package io.github.sspanak.tt9.db.objectbox; - -import androidx.annotation.NonNull; - -import io.github.sspanak.tt9.languages.InvalidLanguageCharactersException; -import io.github.sspanak.tt9.languages.Language; -import io.objectbox.annotation.ConflictStrategy; -import io.objectbox.annotation.Entity; -import io.objectbox.annotation.Id; -import io.objectbox.annotation.Index; -import io.objectbox.annotation.Unique; - -@Entity -public class Word { - @Id public long id; - public int frequency; - public boolean isCustom; - public int langId; - public int length; - @Index public String sequence; - @Index public byte sequenceShort; // up to 2 digits - @Unique(onConflict = ConflictStrategy.REPLACE) public String uniqueId; - public String word; - - public static Word create(@NonNull Language language, @NonNull String word, int frequency) throws InvalidLanguageCharactersException { - Word w = new Word(); - w.frequency = frequency; - w.isCustom = false; - w.langId = language.getId(); - w.length = word.length(); - w.sequence = language.getDigitSequenceForWord(word); - w.sequenceShort = shrinkSequence(w.sequence); - w.uniqueId = (language.getId() + "-" + word); - w.word = word; - - return w; - } - - public static Word create(@NonNull Language language, @NonNull String word, int frequency, boolean isCustom) throws InvalidLanguageCharactersException { - Word w = create(language, word, frequency); - w.isCustom = isCustom; - return w; - } - - public static Byte shrinkSequence(@NonNull String sequence) { - return Byte.parseByte(sequence.substring(0, Math.min(2, sequence.length()))); - } -} diff --git a/src/io/github/sspanak/tt9/db/objectbox/WordList.java b/src/io/github/sspanak/tt9/db/objectbox/WordList.java deleted file mode 100644 index a1446a60..00000000 --- a/src/io/github/sspanak/tt9/db/objectbox/WordList.java +++ /dev/null @@ -1,57 +0,0 @@ -package io.github.sspanak.tt9.db.objectbox; - -import androidx.annotation.NonNull; - -import java.util.ArrayList; -import java.util.List; - - -public class WordList extends ArrayList { - public WordList() { - super(); - } - - - public WordList(@NonNull List words) { - addAll(words); - } - - - @NonNull - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - - for (int i = 0; i < size(); i++) { - sb - .append("word: ").append(get(i).word) - .append(" | sequence: ").append(get(i).sequence) - .append(" | priority: ").append(get(i).frequency) - .append("\n"); - } - - return sb.toString(); - } - - - @NonNull - public WordList filter(int minLength, int minWords) { - WordList filtered = new WordList(); - for (int i = 0; i < size(); i++) { - if (get(i).length == minLength || filtered.size() < minWords) { - filtered.add(get(i)); - } - } - return filtered; - } - - - @NonNull - public ArrayList toStringList() { - ArrayList strings = new ArrayList<>(); - for (int i = 0; i < size(); i++) { - strings.add(get(i).word); - } - return strings; - } -} diff --git a/src/io/github/sspanak/tt9/db/objectbox/WordStore.java b/src/io/github/sspanak/tt9/db/objectbox/WordStore.java deleted file mode 100644 index b27a1e7f..00000000 --- a/src/io/github/sspanak/tt9/db/objectbox/WordStore.java +++ /dev/null @@ -1,208 +0,0 @@ -package io.github.sspanak.tt9.db.objectbox; - -import android.content.Context; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import io.github.sspanak.tt9.Logger; -import io.github.sspanak.tt9.languages.Language; -import io.objectbox.Box; -import io.objectbox.BoxStore; -import io.objectbox.config.DebugFlags; -import io.objectbox.query.Query; -import io.objectbox.query.QueryBuilder; -import io.objectbox.query.QueryCondition; - -public class WordStore { - private BoxStore boxStore; - private Box wordBox; - - private Query longWordQuery; - private Query singleLetterQuery; - - - public WordStore(Context context) { - init(context); - } - - - private void init(Context context) { - boxStore = MyObjectBox.builder() - .androidContext(context.getApplicationContext()) - .debugFlags(Logger.isDebugLevel() ? DebugFlags.LOG_QUERY_PARAMETERS : 0) - .build(); - - wordBox = boxStore.boxFor(Word.class); - - longWordQuery = getLongWordQuery(); - singleLetterQuery = getSingleLetterQuery(); - } - - - private Query getLongWordQuery() { - return boxStore.boxFor(Word.class) - .query() - .equal(Word_.langId, 0) - .startsWith(Word_.sequence, "", QueryBuilder.StringOrder.CASE_SENSITIVE).parameterAlias("seq_start") - .lessOrEqual(Word_.sequence, "", QueryBuilder.StringOrder.CASE_SENSITIVE).parameterAlias("seq_end") - .equal(Word_.sequenceShort, 0) - .startsWith(Word_.word, "", QueryBuilder.StringOrder.CASE_SENSITIVE) - .order(Word_.length) - .orderDesc(Word_.frequency) - .build(); - } - - private Query getSingleLetterQuery() { - return wordBox - .query() - .equal(Word_.langId, 0) - .equal(Word_.sequenceShort, 0) - .startsWith(Word_.word, "", QueryBuilder.StringOrder.CASE_SENSITIVE) - .orderDesc(Word_.frequency) - .build(); - } - - - public long count(int langId) { - try (Query query = wordBox.query(Word_.langId.equal(langId)).build()) { - return query.count(); - } catch (Exception e) { - return 0; - } - } - - - public boolean exists(int langId, @NonNull String word, @NonNull String sequence) { - return get(langId, word, sequence) != null; - } - - - @Nullable - public Word get(int langId, @NonNull String word, @NonNull String sequence) { - QueryCondition where = Word_.langId.equal(langId) - .and(Word_.sequenceShort.equal(Word.shrinkSequence(sequence))) - .and(Word_.word.equal(word, QueryBuilder.StringOrder.CASE_SENSITIVE)); - - try (Query query = wordBox.query(where).build()) { - return query.findFirst(); - } - } - - @Nullable - public List getMany(int langId) { - try (Query query = wordBox.query(Word_.langId.equal(langId)).build()) { - return query.find(); - } - } - - - @NonNull - public WordList getMany(Language language, @NonNull String sequence, @Nullable String filter, int maxWords) { - Query query; - if (sequence.length() < 2) { - singleLetterQuery.setParameter(Word_.sequenceShort, Byte.parseByte(sequence)); - query = singleLetterQuery; - } else { - longWordQuery.setParameter(Word_.sequenceShort, Word.shrinkSequence(sequence)); - longWordQuery.setParameter("seq_start", sequence); - longWordQuery.setParameter("seq_end", sequence + "99"); - query = longWordQuery; - } - - query.setParameter(Word_.langId, language.getId()); - - if (filter != null && !filter.equals("")) { - query.setParameter(Word_.word, filter); - } else { - query.setParameter(Word_.word, ""); - } - - return new WordList(query.find(0, maxWords)); - } - - - public int[] getLanguages() { - try (Query query = wordBox.query().build()) { - return query.property(Word_.langId).distinct().findInts(); - } - } - - - public int getMaxFrequency(int langId) { - return getMaxFrequency(langId, null, null); - } - - - public int getMaxFrequency(int langId, String sequence, String word) { - QueryCondition where = Word_.langId.equal(langId); - - if (sequence != null && word != null) { - where = where.and(Word_.sequenceShort.equal(Word.shrinkSequence(sequence))) - .and(Word_.sequence.equal(sequence)) - .and(Word_.word.notEqual(word)); - } - - try (Query query = wordBox.query(where).build()) { - long max = query.property(Word_.frequency).max(); - return max == Long.MIN_VALUE ? 0 : (int)max; - } - } - - - public void put(@NonNull List words) { - wordBox.put(words); - } - - - public void put(@NonNull Word word) { - wordBox.put(word); - } - - - public WordStore removeMany(@NonNull ArrayList languageIds) { - if (languageIds.size() > 0) { - try (Query query = wordBox.query(Word_.langId.oneOf(IntegerListToIntArray(languageIds))).build()) { - query.remove(); - } - } - - return this; - } - - - public void destroy() { - boxStore.closeThreadResources(); - boxStore.close(); - boxStore.deleteAllFiles(); - } - - - public void runInTransaction(Runnable r) { - boxStore.runInTx(r); - } - - - public void runInTransactionAsync(Runnable action) { - boxStore.runInTxAsync(action, null); - } - - - public void closeThreadResources() { - boxStore.closeThreadResources(); - } - - - private int[] IntegerListToIntArray(ArrayList in) { - int[] out = new int[in.size()]; - Iterator iterator = in.iterator(); - for (int i = 0; i < out.length; i++) { - out[i] = iterator.next(); - } - return out; - } -} diff --git a/src/io/github/sspanak/tt9/db/sqlite/CompiledQueryCache.java b/src/io/github/sspanak/tt9/db/sqlite/CompiledQueryCache.java new file mode 100644 index 00000000..b27afb02 --- /dev/null +++ b/src/io/github/sspanak/tt9/db/sqlite/CompiledQueryCache.java @@ -0,0 +1,63 @@ +package io.github.sspanak.tt9.db.sqlite; + +import android.database.sqlite.SQLiteDatabase; +import android.database.sqlite.SQLiteDoneException; +import android.database.sqlite.SQLiteStatement; + +import androidx.annotation.NonNull; + +import java.util.HashMap; + +class CompiledQueryCache { + private static CompiledQueryCache self; + private final SQLiteDatabase db; + private final HashMap statements = new HashMap<>(); + + private CompiledQueryCache(@NonNull SQLiteDatabase db) { + this.db = db; + } + + CompiledQueryCache execute(String sql) { + get(sql).execute(); + return this; + } + + SQLiteStatement get(@NonNull String sql) { + SQLiteStatement statement = statements.get(sql.hashCode()); + if (statement == null) { + statement = db.compileStatement(sql); + statements.put(sql.hashCode(), statement); + } + + return statement; + } + + long simpleQueryForLong(String sql, long defaultValue) { + try { + return get(sql).simpleQueryForLong(); + } catch (SQLiteDoneException e) { + return defaultValue; + } + } + + + static CompiledQueryCache getInstance(SQLiteDatabase db) { + if (self == null) { + self = new CompiledQueryCache(db); + } + + return self; + } + + static CompiledQueryCache execute(SQLiteDatabase db, String sql) { + return getInstance(db).execute(sql); + } + + static SQLiteStatement get(SQLiteDatabase db, String sql) { + return getInstance(db).get(sql); + } + + static long simpleQueryForLong(SQLiteDatabase db, String sql, long defaultValue) { + return getInstance(db).simpleQueryForLong(sql, defaultValue); + } +} diff --git a/src/io/github/sspanak/tt9/db/sqlite/DeleteOps.java b/src/io/github/sspanak/tt9/db/sqlite/DeleteOps.java new file mode 100644 index 00000000..d8941238 --- /dev/null +++ b/src/io/github/sspanak/tt9/db/sqlite/DeleteOps.java @@ -0,0 +1,10 @@ +package io.github.sspanak.tt9.db.sqlite; + +import androidx.annotation.NonNull; + +public class DeleteOps { + public static void delete(@NonNull SQLiteOpener sqlite, int languageId) { + sqlite.getDb().delete(Tables.getWords(languageId), null, null); + sqlite.getDb().delete(Tables.getWordPositions(languageId), null, null); + } +} diff --git a/src/io/github/sspanak/tt9/db/sqlite/InsertOps.java b/src/io/github/sspanak/tt9/db/sqlite/InsertOps.java new file mode 100644 index 00000000..e747f4fe --- /dev/null +++ b/src/io/github/sspanak/tt9/db/sqlite/InsertOps.java @@ -0,0 +1,79 @@ +package io.github.sspanak.tt9.db.sqlite; + +import android.content.ContentValues; +import android.database.sqlite.SQLiteDatabase; +import android.database.sqlite.SQLiteStatement; + +import androidx.annotation.NonNull; + +import io.github.sspanak.tt9.db.entities.Word; +import io.github.sspanak.tt9.db.entities.WordPosition; +import io.github.sspanak.tt9.languages.Language; + + +public class InsertOps { + private final SQLiteStatement insertWordsQuery; + private final SQLiteStatement insertPositionsQuery; + + + public InsertOps(SQLiteDatabase db, @NonNull Language language) { + // super cache to avoid String concatenation in the dictionary loading loop + insertWordsQuery = CompiledQueryCache.get(db, "INSERT INTO " + Tables.getWords(language.getId()) + " (frequency, position, word) VALUES (?, ?, ?)"); + insertPositionsQuery = CompiledQueryCache.get(db, "INSERT INTO " + Tables.getWordPositions(language.getId()) + " (sequence, `start`, `end`) VALUES (?, ?, ?)"); + } + + + public void insertWord(Word word) { + insertWordsQuery.bindLong(1, word.frequency); + insertWordsQuery.bindLong(2, word.position); + insertWordsQuery.bindString(3, word.word); + insertWordsQuery.execute(); + } + + + public void insertWordPosition(WordPosition position) { + insertPositionsQuery.bindString(1, position.sequence); + insertPositionsQuery.bindLong(2, position.start); + insertPositionsQuery.bindLong(3, position.end); + insertPositionsQuery.execute(); + } + + + public static void insertLanguageMeta(@NonNull SQLiteDatabase db, int langId) { + SQLiteStatement query = CompiledQueryCache.get(db, "REPLACE INTO " + Tables.LANGUAGES_META + " (langId) VALUES (?)"); + query.bindLong(1, langId); + query.execute(); + } + + + public static boolean insertCustomWord(@NonNull SQLiteDatabase db, @NonNull Language language, @NonNull String sequence, @NonNull String word) { + ContentValues values = new ContentValues(); + values.put("langId", language.getId()); + values.put("sequence", sequence); + values.put("word", word); + + long insertId = db.insert(Tables.CUSTOM_WORDS, null, values); + if (insertId == -1) { + return false; + } + + // If the user inserts more than 2^31 custom words, the "position" will overflow and will mess up + // the words table, but realistically it will never happen, so we don't bother preventing it. + + values = new ContentValues(); + values.put("position", (int)-insertId); + values.put("word", word); + insertId = db.insert(Tables.getWords(language.getId()), null, values); + + return insertId != -1; + } + + + public static void restoreCustomWords(@NonNull SQLiteDatabase db, @NonNull Language language) { + CompiledQueryCache.execute( + db, + "INSERT INTO " + Tables.getWords(language.getId()) + " (position, word) " + + "SELECT -id, word FROM " + Tables.CUSTOM_WORDS + " WHERE langId = " + language.getId() + ); + } +} diff --git a/src/io/github/sspanak/tt9/db/sqlite/ReadOps.java b/src/io/github/sspanak/tt9/db/sqlite/ReadOps.java new file mode 100644 index 00000000..9a92bbd6 --- /dev/null +++ b/src/io/github/sspanak/tt9/db/sqlite/ReadOps.java @@ -0,0 +1,223 @@ +package io.github.sspanak.tt9.db.sqlite; + +import android.database.Cursor; +import android.database.sqlite.SQLiteDatabase; +import android.database.sqlite.SQLiteDoneException; +import android.database.sqlite.SQLiteStatement; + +import androidx.annotation.NonNull; + +import io.github.sspanak.tt9.Logger; +import io.github.sspanak.tt9.db.SlowQueryStats; +import io.github.sspanak.tt9.db.entities.WordList; +import io.github.sspanak.tt9.db.entities.WordPositionsStringBuilder; +import io.github.sspanak.tt9.languages.Language; + +public class ReadOps { + private final String LOG_TAG = "ReadOperations"; + + + /** + * Checks if a word exists in the database for the given language (case-insensitive). + */ + public boolean exists(@NonNull SQLiteDatabase db, @NonNull Language language, @NonNull String word) { + String lowercaseWord = word.toLowerCase(language.getLocale()); + String uppercaseWord = word.toUpperCase(language.getLocale()); + + SQLiteStatement query = CompiledQueryCache.get(db, "SELECT COUNT(*) FROM " + Tables.getWords(language.getId()) + " WHERE word IN(?, ?, ?)"); + query.bindString(1, word); + query.bindString(2, lowercaseWord); + query.bindString(3, uppercaseWord); + try { + return query.simpleQueryForLong() > 0; + } catch (SQLiteDoneException e) { + return false; + } + } + + + /** + * Checks if language exists (has words) in the database. + */ + public boolean exists(@NonNull SQLiteDatabase db, int langId) { + return CompiledQueryCache.simpleQueryForLong( + db, + "SELECT COUNT(*) FROM " + Tables.getWords(langId), + 0 + ) > 0; + } + + + @NonNull + public WordList getWords(@NonNull SQLiteDatabase db, @NonNull Language language, @NonNull String positions, String filter, int maximumWords, boolean fullOutput) { + if (positions.isEmpty()) { + Logger.d(LOG_TAG, "No word positions. Not searching words."); + return new WordList(); + } + + + String wordsQuery = getWordsQuery(language, positions, filter, maximumWords, fullOutput); + if (wordsQuery.isEmpty()) { + return new WordList(); + } + + WordList words = new WordList(); + try (Cursor cursor = db.rawQuery(wordsQuery, null)) { + while (cursor.moveToNext()) { + words.add( + cursor.getString(0), + fullOutput ? cursor.getInt(1) : 0, + fullOutput ? cursor.getInt(2) : 0 + ); + } + } + + return words; + } + + + public String getSimilarWordPositions(@NonNull SQLiteDatabase db, @NonNull Language language, @NonNull String sequence, String wordFilter, int minPositions) { + int generations; + + switch (sequence.length()) { + case 2: + generations = wordFilter.isEmpty() ? 1 : Integer.MAX_VALUE; + break; + case 3: + case 4: + generations = wordFilter.isEmpty() ? 2 : Integer.MAX_VALUE; + break; + default: + generations = Integer.MAX_VALUE; + break; + } + + return getWordPositions(db, language, sequence, generations, minPositions, wordFilter); + } + + + @NonNull + public String getWordPositions(@NonNull SQLiteDatabase db, @NonNull Language language, @NonNull String sequence, int generations, int minPositions, String wordFilter) { + if (sequence.length() == 1) { + return sequence; + } + + WordPositionsStringBuilder positions = new WordPositionsStringBuilder(); + + String cachedFactoryPositions = SlowQueryStats.getCachedIfSlow(SlowQueryStats.generateKey(language, sequence, wordFilter, minPositions)); + if (cachedFactoryPositions != null) { + String customWordPositions = getCustomWordPositions(db, language, sequence, generations); + return customWordPositions.isEmpty() ? cachedFactoryPositions : customWordPositions + "," + cachedFactoryPositions; + } + + try (Cursor cursor = db.rawQuery(getPositionsQuery(language, sequence, generations), null)) { + positions.appendFromDbRanges(cursor); + } + + if (positions.size < minPositions) { + Logger.d(LOG_TAG, "Not enough positions: " + positions.size + " < " + minPositions + ". Searching for more."); + try (Cursor cursor = db.rawQuery(getFactoryWordPositionsQuery(language, sequence, Integer.MAX_VALUE), null)) { + positions.appendFromDbRanges(cursor); + } + } + + return positions.toString(); + } + + + + @NonNull private String getCustomWordPositions(@NonNull SQLiteDatabase db, Language language, String sequence, int generations) { + try (Cursor cursor = db.rawQuery(getCustomWordPositionsQuery(language, sequence, generations), null)) { + return new WordPositionsStringBuilder().appendFromDbRanges(cursor).toString(); + } + } + + + private String getPositionsQuery(@NonNull Language language, @NonNull String sequence, int generations) { + return + "SELECT `start`, `end` FROM ( " + + getFactoryWordPositionsQuery(language, sequence, generations) + + ") UNION " + + getCustomWordPositionsQuery(language, sequence, generations); + } + + + + @NonNull private String getFactoryWordPositionsQuery(@NonNull Language language, @NonNull String sequence, int generations) { + StringBuilder sql = new StringBuilder("SELECT `start`, `end` FROM ") + .append(Tables.getWordPositions(language.getId())) + .append(" WHERE "); + + if (generations >= 0 && generations < 10) { + sql.append(" sequence IN(").append(sequence); + + int lastChild = (int)Math.pow(10, generations) - 1; + + for (int seqEnd = 1; seqEnd <= lastChild; seqEnd++) { + if (seqEnd % 10 != 0) { + sql.append(",").append(sequence).append(seqEnd); + } + } + + sql.append(")"); + } else { + sql.append(" sequence = ").append(sequence).append(" OR sequence BETWEEN ").append(sequence).append("1 AND ").append(sequence).append("9"); + sql.append(" ORDER BY `start` "); + sql.append(" LIMIT 100"); + } + + String positionsSql = sql.toString(); + Logger.v(LOG_TAG, "Index SQL: " + positionsSql); + return positionsSql; + } + + + @NonNull private String getCustomWordPositionsQuery(@NonNull Language language, @NonNull String sequence, int generations) { + String sql = "SELECT -id as `start`, -id as `end` FROM " + Tables.CUSTOM_WORDS + + " WHERE langId = " + language.getId() + + " AND (sequence = " + sequence; + + if (generations > 0) { + sql += " OR sequence BETWEEN " + sequence + "1 AND " + sequence + "9)"; + } else { + sql += ")"; + } + + Logger.v(LOG_TAG, "Custom words SQL: " + sql); + return sql; + } + + + @NonNull private String getWordsQuery(@NonNull Language language, @NonNull String positions, @NonNull String filter, int maxWords, boolean fullOutput) { + StringBuilder sql = new StringBuilder(); + sql + .append("SELECT word"); + if (fullOutput) { + sql.append(",frequency,position"); + } + + sql.append(" FROM ").append(Tables.getWords(language.getId())) + .append(" WHERE position IN(").append(positions).append(")"); + + if (!filter.isEmpty()) { + sql.append(" AND word LIKE '").append(filter.replaceAll("'", "''")).append("%'"); + } + + sql + .append(" ORDER BY LENGTH(word), frequency DESC") + .append(" LIMIT ").append(maxWords); + + String wordsSql = sql.toString(); + Logger.v(LOG_TAG, "Words SQL: " + wordsSql); + return wordsSql; + } + + + public int getNextInNormalizationQueue(@NonNull SQLiteDatabase db) { + return (int) CompiledQueryCache.simpleQueryForLong( + db, + "SELECT langId FROM " + Tables.LANGUAGES_META + " WHERE normalizationPending = 1 LIMIT 1", + -1 + ); + } +} diff --git a/src/io/github/sspanak/tt9/db/sqlite/SQLiteOpener.java b/src/io/github/sspanak/tt9/db/sqlite/SQLiteOpener.java new file mode 100644 index 00000000..50854ef6 --- /dev/null +++ b/src/io/github/sspanak/tt9/db/sqlite/SQLiteOpener.java @@ -0,0 +1,85 @@ +package io.github.sspanak.tt9.db.sqlite; + +import android.content.Context; +import android.database.sqlite.SQLiteDatabase; +import android.database.sqlite.SQLiteOpenHelper; + +import java.util.ArrayList; + +import io.github.sspanak.tt9.languages.Language; +import io.github.sspanak.tt9.languages.LanguageCollection; + +public class SQLiteOpener extends SQLiteOpenHelper { + private static final String DATABASE_NAME = "tt9.db"; + private static final int DATABASE_VERSION = 1; + private static SQLiteOpener self; + + private final ArrayList allLanguages; + private SQLiteDatabase db; + + + public SQLiteOpener(Context context) { + super(context, DATABASE_NAME, null, DATABASE_VERSION); + allLanguages = LanguageCollection.getAll(context); + } + + + public static SQLiteOpener getInstance(Context context) { + if (self == null) { + self = new SQLiteOpener(context); + } + + return self; + } + + + @Override + public void onCreate(SQLiteDatabase db) { + for (String query : Tables.getCreateQueries(allLanguages)) { + db.execSQL(query); + } + } + + + @Override + public void onConfigure(SQLiteDatabase db) { + super.onConfigure(db); + setWriteAheadLoggingEnabled(true); + } + + + @Override + public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) { + // No migrations as of now + } + + + public SQLiteDatabase getDb() { + if (db == null) { + db = getWritableDatabase(); + } + return db; + } + + + public void beginTransaction() { + if (db != null) { + db.beginTransactionNonExclusive(); + } + } + + + public void failTransaction() { + if (db != null) { + db.endTransaction(); + } + } + + + public void finishTransaction() { + if (db != null) { + db.setTransactionSuccessful(); + db.endTransaction(); + } + } +} diff --git a/src/io/github/sspanak/tt9/db/sqlite/Tables.java b/src/io/github/sspanak/tt9/db/sqlite/Tables.java new file mode 100644 index 00000000..89fef276 --- /dev/null +++ b/src/io/github/sspanak/tt9/db/sqlite/Tables.java @@ -0,0 +1,111 @@ +package io.github.sspanak.tt9.db.sqlite; + +import android.database.sqlite.SQLiteDatabase; + +import androidx.annotation.NonNull; + +import java.util.ArrayList; + +import io.github.sspanak.tt9.languages.Language; + +public class Tables { + + static final String LANGUAGES_META = "languages_meta"; + static final String CUSTOM_WORDS = "custom_words"; + private static final String POSITIONS_TABLE_BASE_NAME = "word_positions_"; + private static final String WORDS_TABLE_BASE_NAME = "words_"; + + static String getWords(int langId) { return WORDS_TABLE_BASE_NAME + langId; } + static String getWordPositions(int langId) { return POSITIONS_TABLE_BASE_NAME + langId; } + + + static String[] getCreateQueries(ArrayList languages) { + int languageCount = languages.size(); + String[] queries = new String[languageCount * 4 + 3]; + + queries[0] = createCustomWords(); + queries[1] = createCustomWordsIndex(); + queries[2] = createLanguagesMeta(); + + int queryId = 3; + for (Language language : languages) { + queries[queryId++] = createWordsTable(language.getId()); + queries[queryId++] = createWordsIndex(language.getId()); + queries[queryId++] = createWordPositions(language.getId()); + queries[queryId++] = createWordsPositionsIndex(language.getId()); + } + + return queries; + } + + + public static void createWordIndex(@NonNull SQLiteDatabase db, @NonNull Language language) { + CompiledQueryCache.execute(db, createWordsIndex(language.getId())); + } + + public static void createPositionIndex(@NonNull SQLiteDatabase db, @NonNull Language language) { + CompiledQueryCache.execute(db, createWordsPositionsIndex(language.getId())); + } + + + public static void dropIndexes(@NonNull SQLiteDatabase db, @NonNull Language language) { + CompiledQueryCache + .execute(db, dropWordsIndex(language.getId())) + .execute(dropWordPositionsIndex(language.getId())); + } + + + private static String createWordsTable(int langId) { + return + "CREATE TABLE IF NOT EXISTS " + getWords(langId) + " (" + + "frequency INTEGER NOT NULL DEFAULT 0, " + + "position INTEGER NOT NULL, " + + "word TEXT NOT NULL" + + ")"; + } + + private static String createWordsIndex(int langId) { + return "CREATE INDEX IF NOT EXISTS idx_position_" + langId + " ON " + getWords(langId) + " (position, word)"; + } + + private static String dropWordsIndex(int langId) { + return "DROP INDEX IF EXISTS idx_position_" + langId; + } + + private static String createWordPositions(int langId) { + return + "CREATE TABLE IF NOT EXISTS " + getWordPositions(langId) + " (" + + "sequence TEXT NOT NULL, " + + "start INTEGER NOT NULL, " + + "end INTEGER NOT NULL" + + ")"; + } + + private static String createWordsPositionsIndex(int langId) { + return "CREATE INDEX IF NOT EXISTS idx_sequence_start_" + langId + " ON " + getWordPositions(langId) + " (sequence, `start`)"; + } + + private static String dropWordPositionsIndex(int langId) { + return "DROP INDEX IF EXISTS idx_sequence_start_" + langId; + } + + private static String createCustomWords() { + return "CREATE TABLE IF NOT EXISTS " + CUSTOM_WORDS + " (" + + "id INTEGER PRIMARY KEY, " + + "langId INTEGER NOT NULL, " + + "sequence TEXT NOT NULL, " + + "word INTEGER NOT NULL " + + ")"; + } + + private static String createCustomWordsIndex() { + return "CREATE INDEX IF NOT EXISTS idx_langId_sequence ON " + CUSTOM_WORDS + " (langId, sequence)"; + } + + private static String createLanguagesMeta() { + return "CREATE TABLE IF NOT EXISTS " + LANGUAGES_META + " (" + + "langId INTEGER UNIQUE NOT NULL, " + + "normalizationPending INT2 NOT NULL DEFAULT 0 " + + ")"; + } +} diff --git a/src/io/github/sspanak/tt9/db/sqlite/UpdateOps.java b/src/io/github/sspanak/tt9/db/sqlite/UpdateOps.java new file mode 100644 index 00000000..94d8ee3f --- /dev/null +++ b/src/io/github/sspanak/tt9/db/sqlite/UpdateOps.java @@ -0,0 +1,59 @@ +package io.github.sspanak.tt9.db.sqlite; + +import android.database.sqlite.SQLiteDatabase; +import android.database.sqlite.SQLiteStatement; + +import androidx.annotation.NonNull; + +import io.github.sspanak.tt9.Logger; +import io.github.sspanak.tt9.languages.Language; +import io.github.sspanak.tt9.preferences.SettingsStore; + + +public class UpdateOps { + private static final String LOG_TAG = UpdateOps.class.getSimpleName(); + + + public static boolean changeFrequency(@NonNull SQLiteDatabase db, @NonNull Language language, String word, int position, int frequency) { + String sql = "UPDATE " + Tables.getWords(language.getId()) + " SET frequency = ? WHERE position = ?"; + + if (word != null && !word.isEmpty()) { + sql += " AND word = ?"; + } + + SQLiteStatement query = CompiledQueryCache.get(db, sql); + query.bindLong(1, frequency); + query.bindLong(2, position); + if (word != null && !word.isEmpty()) { + query.bindString(3, word); + } + + Logger.v(LOG_TAG, "Change frequency SQL: " + query + "; (" + frequency + ", " + position + ", " + word + ")"); + + return query.executeUpdateDelete() > 0; + } + + + public static void normalize(@NonNull SQLiteDatabase db, int langId) { + if (langId <= 0) { + return; + } + + SQLiteStatement query = CompiledQueryCache.get(db, "UPDATE " + Tables.getWords(langId) + " SET frequency = frequency / ?"); + query.bindLong(1, SettingsStore.WORD_FREQUENCY_NORMALIZATION_DIVIDER); + query.execute(); + + query = CompiledQueryCache.get(db, "UPDATE " + Tables.LANGUAGES_META + " SET normalizationPending = ? WHERE langId = ?"); + query.bindLong(1, 0); + query.bindLong(2, langId); + query.execute(); + } + + + public static void scheduleNormalization(@NonNull SQLiteDatabase db, @NonNull Language language) { + SQLiteStatement query = CompiledQueryCache.get(db, "UPDATE " + Tables.LANGUAGES_META + " SET normalizationPending = ? WHERE langId = ?"); + query.bindLong(1, 1); + query.bindLong(2, language.getId()); + query.execute(); + } +} diff --git a/src/io/github/sspanak/tt9/ime/EmptyDatabaseWarning.java b/src/io/github/sspanak/tt9/ime/EmptyDatabaseWarning.java index b2febcf0..7ebc2a58 100644 --- a/src/io/github/sspanak/tt9/ime/EmptyDatabaseWarning.java +++ b/src/io/github/sspanak/tt9/ime/EmptyDatabaseWarning.java @@ -5,22 +5,19 @@ import android.content.Context; import java.util.HashMap; import io.github.sspanak.tt9.R; -import io.github.sspanak.tt9.db.DictionaryDb; +import io.github.sspanak.tt9.db.WordStoreAsync; import io.github.sspanak.tt9.languages.Language; import io.github.sspanak.tt9.languages.LanguageCollection; import io.github.sspanak.tt9.preferences.SettingsStore; import io.github.sspanak.tt9.ui.UI; public class EmptyDatabaseWarning { - final int WARNING_INTERVAL; private static final HashMap warningDisplayedTime = new HashMap<>(); private Context context; private Language language; - public EmptyDatabaseWarning(SettingsStore settings) { - WARNING_INTERVAL = settings.getDictionaryMissingWarningInterval(); - + public EmptyDatabaseWarning() { for (Language lang : LanguageCollection.getAll(context)) { if (!warningDisplayedTime.containsKey(lang.getId())) { warningDisplayedTime.put(lang.getId(), 0L); @@ -33,7 +30,7 @@ public class EmptyDatabaseWarning { this.language = language; if (isItTimeAgain(TraditionalT9.getMainContext())) { - DictionaryDb.areThereWords(this::show, language); + WordStoreAsync.areThereWords(this::show, language); } } @@ -44,7 +41,7 @@ public class EmptyDatabaseWarning { long now = System.currentTimeMillis(); Long lastWarningTime = warningDisplayedTime.get(language.getId()); - return lastWarningTime != null && now - lastWarningTime > WARNING_INTERVAL; + return lastWarningTime != null && now - lastWarningTime > SettingsStore.DICTIONARY_MISSING_WARNING_INTERVAL; } private void show(boolean areThereWords) { diff --git a/src/io/github/sspanak/tt9/ime/TraditionalT9.java b/src/io/github/sspanak/tt9/ime/TraditionalT9.java index e4dad23e..dd0e479b 100644 --- a/src/io/github/sspanak/tt9/ime/TraditionalT9.java +++ b/src/io/github/sspanak/tt9/ime/TraditionalT9.java @@ -18,7 +18,7 @@ import java.util.List; import io.github.sspanak.tt9.Logger; import io.github.sspanak.tt9.R; -import io.github.sspanak.tt9.db.DictionaryDb; +import io.github.sspanak.tt9.db.WordStoreAsync; import io.github.sspanak.tt9.ime.helpers.AppHacks; import io.github.sspanak.tt9.ime.helpers.InputModeValidator; import io.github.sspanak.tt9.ime.helpers.InputType; @@ -41,6 +41,7 @@ public class TraditionalT9 extends KeyPadHandler { @NonNull private TextField textField = new TextField(null, null); @NonNull private InputType inputType = new InputType(null, null); @NonNull private final Handler autoAcceptHandler = new Handler(Looper.getMainLooper()); + @NonNull private final Handler normalizationHandler = new Handler(Looper.getMainLooper()); // input mode private ArrayList allowedInputModes = new ArrayList<>(); @@ -149,8 +150,7 @@ public class TraditionalT9 extends KeyPadHandler { self = this; Logger.enableDebugLevel(settings.getDebugLogsEnabled()); - DictionaryDb.init(this); - DictionaryDb.normalizeWordFrequencies(settings); + WordStoreAsync.init(this); if (mainView == null) { mainView = new MainView(this); @@ -219,6 +219,7 @@ public class TraditionalT9 extends KeyPadHandler { return; } + normalizationHandler.removeCallbacksAndMessages(null); initUi(); updateInputViewShown(); } @@ -234,6 +235,9 @@ public class TraditionalT9 extends KeyPadHandler { onFinishTyping(); clearSuggestions(); statusBar.setText("--"); + + normalizationHandler.removeCallbacksAndMessages(null); + normalizationHandler.postDelayed(WordStoreAsync::normalizeNext, SettingsStore.WORD_NORMALIZATION_DELAY); } diff --git a/src/io/github/sspanak/tt9/ime/modes/ModePredictive.java b/src/io/github/sspanak/tt9/ime/modes/ModePredictive.java index 1aa108db..06be4c49 100644 --- a/src/io/github/sspanak/tt9/ime/modes/ModePredictive.java +++ b/src/io/github/sspanak/tt9/ime/modes/ModePredictive.java @@ -6,7 +6,7 @@ import java.util.ArrayList; import io.github.sspanak.tt9.Logger; import io.github.sspanak.tt9.TextTools; -import io.github.sspanak.tt9.db.DictionaryDb; +import io.github.sspanak.tt9.db.WordStoreAsync; import io.github.sspanak.tt9.ime.helpers.InputType; import io.github.sspanak.tt9.ime.helpers.TextField; import io.github.sspanak.tt9.ime.modes.helpers.AutoSpace; @@ -271,7 +271,7 @@ public class ModePredictive extends InputMode { // emoji and punctuation are not in the database, so there is no point in // running queries that would update nothing if (!sequence.startsWith("11") && !sequence.equals("1") && !sequence.startsWith("0")) { - DictionaryDb.incrementWordFrequency(language, currentWord, sequence); + WordStoreAsync.makeTopWord(language, currentWord, sequence); } } catch (Exception e) { Logger.e(LOG_TAG, "Failed incrementing priority of word: '" + currentWord + "'. " + e.getMessage()); diff --git a/src/io/github/sspanak/tt9/ime/modes/helpers/Predictions.java b/src/io/github/sspanak/tt9/ime/modes/helpers/Predictions.java index f268d322..d4e8874e 100644 --- a/src/io/github/sspanak/tt9/ime/modes/helpers/Predictions.java +++ b/src/io/github/sspanak/tt9/ime/modes/helpers/Predictions.java @@ -3,7 +3,7 @@ package io.github.sspanak.tt9.ime.modes.helpers; import java.util.ArrayList; import java.util.regex.Pattern; -import io.github.sspanak.tt9.db.DictionaryDb; +import io.github.sspanak.tt9.db.WordStoreAsync; import io.github.sspanak.tt9.ime.EmptyDatabaseWarning; import io.github.sspanak.tt9.languages.Characters; import io.github.sspanak.tt9.languages.Language; @@ -32,7 +32,7 @@ public class Predictions { public Predictions(SettingsStore settingsStore) { - emptyDbWarning = new EmptyDatabaseWarning(settingsStore); + emptyDbWarning = new EmptyDatabaseWarning(); settings = settingsStore; // digitSequence limiter when selecting emoji @@ -128,13 +128,13 @@ public class Predictions { if (loadStatic()) { onWordsChanged.run(); } else { - DictionaryDb.getWords( + WordStoreAsync.getWords( (words) -> onDbWords(words, true), language, digitSequence, stem, - settings.getSuggestionsMin(), - settings.getSuggestionsMax() + SettingsStore.SUGGESTIONS_MIN, + SettingsStore.SUGGESTIONS_MAX ); } } @@ -176,7 +176,7 @@ public class Predictions { } private void loadWithoutLeadingPunctuation() { - DictionaryDb.getWords( + WordStoreAsync.getWords( (dbWords) -> { char firstChar = inputWord.charAt(0); for (int i = 0; i < dbWords.size(); i++) { @@ -187,8 +187,8 @@ public class Predictions { language, digitSequence.substring(1), stem.length() > 1 ? stem.substring(1) : "", - settings.getSuggestionsMin(), - settings.getSuggestionsMax() + SettingsStore.SUGGESTIONS_MIN, + SettingsStore.SUGGESTIONS_MAX ); } diff --git a/src/io/github/sspanak/tt9/languages/Language.java b/src/io/github/sspanak/tt9/languages/Language.java index 3ed1a18d..896f4d9e 100644 --- a/src/io/github/sspanak/tt9/languages/Language.java +++ b/src/io/github/sspanak/tt9/languages/Language.java @@ -101,7 +101,6 @@ public class Language { return keyChars; } - final public int getId() { if (id == 0) { id = generateId(); @@ -172,8 +171,8 @@ public class Language { /** * generateId * Uses the letters of the Locale to generate an ID for the language. - * Each letter is converted to uppercase and used as n 5-bit integer. Then the the 5-bits - * are packed to form a 10-bit or a 20-bit integer, depending on the Locale. + * Each letter is converted to uppercase and used as a 5-bit integer. Then the 5-bits + * are packed to form a 10-bit or a 20-bit integer, depending on the Locale length. * * Example (2-letter Locale) * "en" @@ -186,12 +185,14 @@ public class Language { * -> "B" | "G" | "B" | "G" * -> 2 | 224 | 2048 | 229376 (shift each 5-bit number, not overlap with the previous ones) * -> 231650 + * + * Maximum ID is: "zz-ZZ" -> 879450 */ private int generateId() { String idString = (locale.getLanguage() + locale.getCountry()).toUpperCase(); int idInt = 0; for (int i = 0; i < idString.length(); i++) { - idInt |= ((idString.charAt(i) & 31) << (i * 5)); + idInt |= ((idString.codePointAt(i) & 31) << (i * 5)); } return idInt; diff --git a/src/io/github/sspanak/tt9/languages/LanguageCollection.java b/src/io/github/sspanak/tt9/languages/LanguageCollection.java index dd3feb07..a25d8af1 100644 --- a/src/io/github/sspanak/tt9/languages/LanguageCollection.java +++ b/src/io/github/sspanak/tt9/languages/LanguageCollection.java @@ -95,7 +95,6 @@ public class LanguageCollection { return getAll(context,false); } - public static String toString(ArrayList list) { StringBuilder stringList = new StringBuilder(); int listSize = list.size(); diff --git a/src/io/github/sspanak/tt9/preferences/PreferencesActivity.java b/src/io/github/sspanak/tt9/preferences/PreferencesActivity.java index 3b712122..995d11af 100644 --- a/src/io/github/sspanak/tt9/preferences/PreferencesActivity.java +++ b/src/io/github/sspanak/tt9/preferences/PreferencesActivity.java @@ -15,9 +15,9 @@ import androidx.preference.PreferenceFragmentCompat; import io.github.sspanak.tt9.Logger; import io.github.sspanak.tt9.R; -import io.github.sspanak.tt9.db.DictionaryDb; +import io.github.sspanak.tt9.db.WordStoreAsync; import io.github.sspanak.tt9.db.DictionaryLoader; -import io.github.sspanak.tt9.db.SQLWords; +import io.github.sspanak.tt9.db.LegacyDb; import io.github.sspanak.tt9.ime.helpers.GlobalKeyboardSettings; import io.github.sspanak.tt9.ime.helpers.InputModeValidator; import io.github.sspanak.tt9.preferences.helpers.Hotkeys; @@ -28,6 +28,7 @@ import io.github.sspanak.tt9.preferences.screens.HotkeysScreen; import io.github.sspanak.tt9.preferences.screens.KeyPadScreen; import io.github.sspanak.tt9.preferences.screens.MainSettingsScreen; import io.github.sspanak.tt9.preferences.screens.SetupScreen; +import io.github.sspanak.tt9.preferences.screens.UsageStatsScreen; import io.github.sspanak.tt9.ui.DictionaryLoadingBar; public class PreferencesActivity extends AppCompatActivity implements PreferenceFragmentCompat.OnPreferenceStartFragmentCallback { @@ -42,9 +43,8 @@ public class PreferencesActivity extends AppCompatActivity implements Preference applyTheme(); Logger.enableDebugLevel(settings.getDebugLogsEnabled()); - new SQLWords(this).clear(); - DictionaryDb.init(this); - DictionaryDb.normalizeWordFrequencies(settings); + try (LegacyDb db = new LegacyDb(this)) { db.clear(); } + WordStoreAsync.init(this); InputModeValidator.validateEnabledLanguages(this, settings.getEnabledLanguageIds()); validateFunctionKeys(); @@ -99,6 +99,8 @@ public class PreferencesActivity extends AppCompatActivity implements Preference return new KeyPadScreen(this); case "Setup": return new SetupScreen(this); + case "SlowQueries": + return new UsageStatsScreen(this); default: return new MainSettingsScreen(this); } diff --git a/src/io/github/sspanak/tt9/preferences/SettingsStore.java b/src/io/github/sspanak/tt9/preferences/SettingsStore.java index 48701078..1c518e5a 100644 --- a/src/io/github/sspanak/tt9/preferences/SettingsStore.java +++ b/src/io/github/sspanak/tt9/preferences/SettingsStore.java @@ -272,21 +272,19 @@ public class SettingsStore { public boolean getDebugLogsEnabled() { return prefs.getBoolean("pref_enable_debug_logs", Logger.isDebugLevel()); } - public int getDictionaryImportProgressUpdateInterval() { return 250; /* ms */ } - public int getDictionaryImportWordChunkSize() { return 1000; /* words */ } - - public int getDictionaryMissingWarningInterval() { return 30000; /* ms */ } - - public int getSuggestionsMax() { return 20; } - public int getSuggestionsMin() { return 8; } - - public int getSuggestionSelectAnimationDuration() { return 66; } - public int getSuggestionTranslateAnimationDuration() { return 0; } - - public int getSoftKeyRepeatDelay() { return 40; /* ms */ } - - public int getWordFrequencyMax() { return 25500; } - public int getWordFrequencyNormalizationDivider() { return 100; } // normalized frequency = getWordFrequencyMax() / getWordFrequencyNormalizationDivider() + public final static int DICTIONARY_IMPORT_PROGRESS_UPDATE_BATCH_SIZE = 1000; // items + public final static int DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME = 250; // ms + public final static int DICTIONARY_MISSING_WARNING_INTERVAL = 30000; // ms + public final static int PREFERENCES_CLICK_DEBOUNCE_TIME = 250; // ms + public final static byte SLOW_QUERY_TIME = 50; // ms + public final static int SOFT_KEY_REPEAT_DELAY = 40; // ms + public final static int SUGGESTIONS_MAX = 20; + public final static int SUGGESTIONS_MIN = 8; + public final static int SUGGESTIONS_SELECT_ANIMATION_DURATION = 66; + public final static int SUGGESTIONS_TRANSLATE_ANIMATION_DURATION = 0; + public final static int WORD_FREQUENCY_MAX = 25500; + public final static int WORD_FREQUENCY_NORMALIZATION_DIVIDER = 100; // normalized frequency = WORD_FREQUENCY_MAX / WORD_FREQUENCY_NORMALIZATION_DIVIDER + public final static int WORD_NORMALIZATION_DELAY = 120000; // ms /************* hack settings *************/ diff --git a/src/io/github/sspanak/tt9/preferences/items/ItemClickable.java b/src/io/github/sspanak/tt9/preferences/items/ItemClickable.java index c1788cb1..d1cfec36 100644 --- a/src/io/github/sspanak/tt9/preferences/items/ItemClickable.java +++ b/src/io/github/sspanak/tt9/preferences/items/ItemClickable.java @@ -6,9 +6,9 @@ import java.util.ArrayList; import java.util.List; import io.github.sspanak.tt9.Logger; +import io.github.sspanak.tt9.preferences.SettingsStore; abstract public class ItemClickable { - private final int CLICK_DEBOUNCE_TIME = 250; private long lastClickTime = 0; protected final Preference item; @@ -69,7 +69,7 @@ abstract public class ItemClickable { */ protected boolean debounceClick(Preference p) { long now = System.currentTimeMillis(); - if (now - lastClickTime < CLICK_DEBOUNCE_TIME) { + if (now - lastClickTime < SettingsStore.PREFERENCES_CLICK_DEBOUNCE_TIME) { Logger.d("debounceClick", "Preference click debounced."); return true; } diff --git a/src/io/github/sspanak/tt9/preferences/items/ItemDropDown.java b/src/io/github/sspanak/tt9/preferences/items/ItemDropDown.java index e95e9353..e2440ddb 100644 --- a/src/io/github/sspanak/tt9/preferences/items/ItemDropDown.java +++ b/src/io/github/sspanak/tt9/preferences/items/ItemDropDown.java @@ -60,13 +60,11 @@ public class ItemDropDown { } } - public ItemDropDown preview() { + public void preview() { try { setPreview(values.get(Integer.parseInt(item.getValue()))); } catch (NumberFormatException e) { setPreview(""); } - - return this; } } diff --git a/src/io/github/sspanak/tt9/preferences/items/ItemTruncateAll.java b/src/io/github/sspanak/tt9/preferences/items/ItemTruncateAll.java index e35fdf88..37cae994 100644 --- a/src/io/github/sspanak/tt9/preferences/items/ItemTruncateAll.java +++ b/src/io/github/sspanak/tt9/preferences/items/ItemTruncateAll.java @@ -2,9 +2,13 @@ package io.github.sspanak.tt9.preferences.items; import androidx.preference.Preference; +import java.util.ArrayList; + import io.github.sspanak.tt9.R; -import io.github.sspanak.tt9.db.DictionaryDb; +import io.github.sspanak.tt9.db.WordStoreAsync; import io.github.sspanak.tt9.db.DictionaryLoader; +import io.github.sspanak.tt9.languages.Language; +import io.github.sspanak.tt9.languages.LanguageCollection; import io.github.sspanak.tt9.preferences.PreferencesActivity; import io.github.sspanak.tt9.ui.UI; @@ -30,7 +34,11 @@ public class ItemTruncateAll extends ItemClickable { } onStartDeleting(); - DictionaryDb.deleteWords(activity.getApplicationContext(), this::onFinishDeleting); + ArrayList languageIds = new ArrayList<>(); + for (Language lang : LanguageCollection.getAll(activity, false)) { + languageIds.add(lang.getId()); + } + WordStoreAsync.deleteWords(this::onFinishDeleting, languageIds); return true; } diff --git a/src/io/github/sspanak/tt9/preferences/items/ItemTruncateUnselected.java b/src/io/github/sspanak/tt9/preferences/items/ItemTruncateUnselected.java index 86787aad..bdb05158 100644 --- a/src/io/github/sspanak/tt9/preferences/items/ItemTruncateUnselected.java +++ b/src/io/github/sspanak/tt9/preferences/items/ItemTruncateUnselected.java @@ -4,7 +4,7 @@ import androidx.preference.Preference; import java.util.ArrayList; -import io.github.sspanak.tt9.db.DictionaryDb; +import io.github.sspanak.tt9.db.WordStoreAsync; import io.github.sspanak.tt9.db.DictionaryLoader; import io.github.sspanak.tt9.languages.Language; import io.github.sspanak.tt9.languages.LanguageCollection; @@ -40,7 +40,7 @@ public class ItemTruncateUnselected extends ItemTruncateAll { } onStartDeleting(); - DictionaryDb.deleteWords(this::onFinishDeleting, unselectedLanguageIds); + WordStoreAsync.deleteWords(this::onFinishDeleting, unselectedLanguageIds); return true; } diff --git a/src/io/github/sspanak/tt9/preferences/screens/DictionariesScreen.java b/src/io/github/sspanak/tt9/preferences/screens/DictionariesScreen.java index 217d5c53..6b42660a 100644 --- a/src/io/github/sspanak/tt9/preferences/screens/DictionariesScreen.java +++ b/src/io/github/sspanak/tt9/preferences/screens/DictionariesScreen.java @@ -11,8 +11,6 @@ import io.github.sspanak.tt9.preferences.items.ItemTruncateUnselected; public class DictionariesScreen extends BaseScreenFragment { private ItemLoadDictionary loadItem; - private ItemTruncateUnselected deleteItem; - private ItemTruncateAll truncateItem; public DictionariesScreen() { init(); } public DictionariesScreen(PreferencesActivity activity) { init(activity); } @@ -37,14 +35,14 @@ public class DictionariesScreen extends BaseScreenFragment { activity.getDictionaryProgressBar() ); - deleteItem = new ItemTruncateUnselected( + ItemTruncateUnselected deleteItem = new ItemTruncateUnselected( findPreference(ItemTruncateUnselected.NAME), activity, activity.settings, activity.getDictionaryLoader() ); - truncateItem = new ItemTruncateAll( + ItemTruncateAll truncateItem = new ItemTruncateAll( findPreference(ItemTruncateAll.NAME), activity, activity.getDictionaryLoader() diff --git a/src/io/github/sspanak/tt9/preferences/screens/UsageStatsScreen.java b/src/io/github/sspanak/tt9/preferences/screens/UsageStatsScreen.java new file mode 100644 index 00000000..abd17a21 --- /dev/null +++ b/src/io/github/sspanak/tt9/preferences/screens/UsageStatsScreen.java @@ -0,0 +1,74 @@ +package io.github.sspanak.tt9.preferences.screens; + +import android.content.ClipData; +import android.content.ClipboardManager; +import android.content.Context; +import android.os.Build; + +import androidx.preference.Preference; + +import io.github.sspanak.tt9.R; +import io.github.sspanak.tt9.db.SlowQueryStats; +import io.github.sspanak.tt9.preferences.PreferencesActivity; +import io.github.sspanak.tt9.ui.UI; + +public class UsageStatsScreen extends BaseScreenFragment { + private final static String RESET_BUTTON = "pref_slow_queries_reset_stats"; + private final static String SUMMARY_CONTAINER = "summary_container"; + private final static String QUERY_LIST_CONTAINER = "query_list_container"; + + public UsageStatsScreen() { init(); } + public UsageStatsScreen(PreferencesActivity activity) { init(activity); } + + @Override protected int getTitle() { return R.string.pref_category_usage_stats; } + @Override protected int getXml() { return R.xml.prefs_screen_usage_stats; } + + @Override + protected void onCreate() { + printSummary(); + printSlowQueries(); + enableLogsCopy(); + + Preference resetButton = findPreference(RESET_BUTTON); + if (resetButton != null) { + resetButton.setOnPreferenceClickListener((Preference p) -> { + SlowQueryStats.clear(); + printSummary(); + printSlowQueries(); + return true; + }); + } + } + + private void printSummary() { + Preference logsContainer = findPreference(SUMMARY_CONTAINER); + if (logsContainer != null) { + logsContainer.setSummary(SlowQueryStats.getSummary()); + } + } + + private void printSlowQueries() { + Preference queryListContainer = findPreference(QUERY_LIST_CONTAINER); + if (queryListContainer != null) { + String slowQueries = SlowQueryStats.getList(); + queryListContainer.setSummary(slowQueries.isEmpty() ? "No slow queries." : slowQueries); + } + } + + + private void enableLogsCopy() { + Preference queryListContainer = findPreference(QUERY_LIST_CONTAINER); + if (activity == null || queryListContainer == null) { + return; + } + + ClipboardManager clipboard = (ClipboardManager) activity.getSystemService(Context.CLIPBOARD_SERVICE); + queryListContainer.setOnPreferenceClickListener((Preference p) -> { + clipboard.setPrimaryClip(ClipData.newPlainText("TT9 debug log", p.getSummary())); + if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.S_V2) { + UI.toast(activity, "Logs copied."); + } + return true; + }); + } +} diff --git a/src/io/github/sspanak/tt9/ui/AddWordAct.java b/src/io/github/sspanak/tt9/ui/AddWordAct.java index 91b9565b..26e10712 100644 --- a/src/io/github/sspanak/tt9/ui/AddWordAct.java +++ b/src/io/github/sspanak/tt9/ui/AddWordAct.java @@ -9,14 +9,18 @@ import androidx.appcompat.app.AppCompatActivity; import io.github.sspanak.tt9.Logger; import io.github.sspanak.tt9.R; -import io.github.sspanak.tt9.db.DictionaryDb; -import io.github.sspanak.tt9.db.exceptions.InsertBlankWordException; +import io.github.sspanak.tt9.db.WordStoreAsync; import io.github.sspanak.tt9.ime.TraditionalT9; -import io.github.sspanak.tt9.languages.InvalidLanguageException; import io.github.sspanak.tt9.languages.Language; import io.github.sspanak.tt9.languages.LanguageCollection; public class AddWordAct extends AppCompatActivity { + public static final int CODE_SUCCESS = 0; + public static final int CODE_BLANK_WORD = 1; + public static final int CODE_INVALID_LANGUAGE = 2; + public static final int CODE_WORD_EXISTS = 3; + public static final int CODE_GENERAL_ERROR = 666; + public static final String INTENT_FILTER = "tt9.add_word"; private Language language; @@ -61,14 +65,22 @@ public class AddWordAct extends AppCompatActivity { private void onAddedWord(int statusCode) { String message; switch (statusCode) { - case 0: + case CODE_SUCCESS: message = getString(R.string.add_word_success, word); break; - case 1: + case CODE_WORD_EXISTS: message = getResources().getString(R.string.add_word_exist, word); break; + case CODE_BLANK_WORD: + message = getString(R.string.add_word_blank); + break; + + case CODE_INVALID_LANGUAGE: + message = getResources().getString(R.string.add_word_invalid_language); + break; + default: message = getString(R.string.error_unexpected); break; @@ -80,22 +92,7 @@ public class AddWordAct extends AppCompatActivity { public void addWord(View v) { - try { - Logger.d("addWord", "Attempting to add word: '" + word + "'..."); - DictionaryDb.insertWord(this::onAddedWord, language, word); - } catch (InsertBlankWordException e) { - Logger.e("AddWordAct.addWord", e.getMessage()); - finish(); - sendMessageToMain(getString(R.string.add_word_blank)); - } catch (InvalidLanguageException e) { - Logger.e("AddWordAct.addWord", "Cannot insert a word for language: '" + language.getName() + "'. " + e.getMessage()); - finish(); - sendMessageToMain(getString(R.string.add_word_invalid_language)); - } catch (Exception e) { - Logger.e("AddWordAct.addWord", e.getMessage()); - finish(); - sendMessageToMain(e.getMessage()); - } + WordStoreAsync.put(this::onAddedWord, language, word); } diff --git a/src/io/github/sspanak/tt9/ui/main/BaseMainLayout.java b/src/io/github/sspanak/tt9/ui/main/BaseMainLayout.java index e50a9fe3..948cb2ea 100644 --- a/src/io/github/sspanak/tt9/ui/main/BaseMainLayout.java +++ b/src/io/github/sspanak/tt9/ui/main/BaseMainLayout.java @@ -15,7 +15,7 @@ abstract class BaseMainLayout { protected View view = null; protected ArrayList keys = new ArrayList<>(); - public BaseMainLayout(TraditionalT9 tt9, int xml) { + BaseMainLayout(TraditionalT9 tt9, int xml) { this.tt9 = tt9; this.xml = xml; } diff --git a/src/io/github/sspanak/tt9/ui/main/MainLayoutNumpad.java b/src/io/github/sspanak/tt9/ui/main/MainLayoutNumpad.java index 941164be..30476b99 100644 --- a/src/io/github/sspanak/tt9/ui/main/MainLayoutNumpad.java +++ b/src/io/github/sspanak/tt9/ui/main/MainLayoutNumpad.java @@ -13,7 +13,7 @@ import io.github.sspanak.tt9.ime.TraditionalT9; import io.github.sspanak.tt9.ui.main.keys.SoftKey; class MainLayoutNumpad extends BaseMainLayout { - public MainLayoutNumpad(TraditionalT9 tt9) { + MainLayoutNumpad(TraditionalT9 tt9) { super(tt9, R.layout.main_numpad); } diff --git a/src/io/github/sspanak/tt9/ui/main/MainLayoutSmall.java b/src/io/github/sspanak/tt9/ui/main/MainLayoutSmall.java index ec0d201a..4d9a87a5 100644 --- a/src/io/github/sspanak/tt9/ui/main/MainLayoutSmall.java +++ b/src/io/github/sspanak/tt9/ui/main/MainLayoutSmall.java @@ -12,7 +12,7 @@ import io.github.sspanak.tt9.ime.TraditionalT9; import io.github.sspanak.tt9.ui.main.keys.SoftKey; class MainLayoutSmall extends BaseMainLayout { - public MainLayoutSmall(TraditionalT9 tt9) { + MainLayoutSmall(TraditionalT9 tt9) { super(tt9, R.layout.main_small); } diff --git a/src/io/github/sspanak/tt9/ui/main/keys/SoftKey.java b/src/io/github/sspanak/tt9/ui/main/keys/SoftKey.java index 5e0dbf11..b44aca86 100644 --- a/src/io/github/sspanak/tt9/ui/main/keys/SoftKey.java +++ b/src/io/github/sspanak/tt9/ui/main/keys/SoftKey.java @@ -17,6 +17,7 @@ import androidx.core.content.ContextCompat; import io.github.sspanak.tt9.Logger; import io.github.sspanak.tt9.R; import io.github.sspanak.tt9.ime.TraditionalT9; +import io.github.sspanak.tt9.preferences.SettingsStore; public class SoftKey extends androidx.appcompat.widget.AppCompatButton implements View.OnTouchListener, View.OnLongClickListener { protected TraditionalT9 tt9; @@ -109,7 +110,7 @@ public class SoftKey extends androidx.appcompat.widget.AppCompatButton implement handleHold(); lastPressedKey = getId(); repeatHandler.removeCallbacks(this::repeatOnLongPress); - repeatHandler.postDelayed(this::repeatOnLongPress, tt9.getSettings().getSoftKeyRepeatDelay()); + repeatHandler.postDelayed(this::repeatOnLongPress, SettingsStore.SOFT_KEY_REPEAT_DELAY); } } diff --git a/src/io/github/sspanak/tt9/ui/tray/SuggestionsBar.java b/src/io/github/sspanak/tt9/ui/tray/SuggestionsBar.java index 5794e715..18389a3b 100644 --- a/src/io/github/sspanak/tt9/ui/tray/SuggestionsBar.java +++ b/src/io/github/sspanak/tt9/ui/tray/SuggestionsBar.java @@ -19,6 +19,7 @@ import java.util.List; import io.github.sspanak.tt9.R; import io.github.sspanak.tt9.ime.TraditionalT9; +import io.github.sspanak.tt9.preferences.SettingsStore; public class SuggestionsBar { private final List suggestions = new ArrayList<>(); @@ -52,13 +53,10 @@ public class SuggestionsBar { private void configureAnimation() { DefaultItemAnimator animator = new DefaultItemAnimator(); - int translateDuration = tt9.getSettings().getSuggestionTranslateAnimationDuration(); - int selectDuration = tt9.getSettings().getSuggestionSelectAnimationDuration(); - - animator.setMoveDuration(selectDuration); - animator.setChangeDuration(translateDuration); - animator.setAddDuration(translateDuration); - animator.setRemoveDuration(translateDuration); + animator.setMoveDuration(SettingsStore.SUGGESTIONS_SELECT_ANIMATION_DURATION); + animator.setChangeDuration(SettingsStore.SUGGESTIONS_TRANSLATE_ANIMATION_DURATION); + animator.setAddDuration(SettingsStore.SUGGESTIONS_TRANSLATE_ANIMATION_DURATION); + animator.setRemoveDuration(SettingsStore.SUGGESTIONS_TRANSLATE_ANIMATION_DURATION); mView.setItemAnimator(animator); }