From 2898db0702bb2517f71b373e27c73835fcb84578 Mon Sep 17 00:00:00 2001 From: sspanak Date: Sat, 21 Dec 2024 17:28:55 +0200 Subject: [PATCH] optimized the long positions caching --- .../tt9/db/entities/LongPositionsCache.java | 44 ------------------- .../sspanak/tt9/db/sqlite/InsertOps.java | 5 ++- .../sspanak/tt9/db/sqlite/Migration.java | 4 ++ .../github/sspanak/tt9/db/sqlite/ReadOps.java | 29 ++++-------- .../github/sspanak/tt9/db/sqlite/Tables.java | 3 +- .../tt9/db/words/DictionaryLoader.java | 4 +- 6 files changed, 21 insertions(+), 68 deletions(-) delete mode 100644 app/src/main/java/io/github/sspanak/tt9/db/entities/LongPositionsCache.java diff --git a/app/src/main/java/io/github/sspanak/tt9/db/entities/LongPositionsCache.java b/app/src/main/java/io/github/sspanak/tt9/db/entities/LongPositionsCache.java deleted file mode 100644 index 47dec90a..00000000 --- a/app/src/main/java/io/github/sspanak/tt9/db/entities/LongPositionsCache.java +++ /dev/null @@ -1,44 +0,0 @@ -package io.github.sspanak.tt9.db.entities; - -import androidx.annotation.NonNull; - -import java.util.HashMap; - -import io.github.sspanak.tt9.languages.Language; -import io.github.sspanak.tt9.preferences.settings.SettingsStore; - -public class LongPositionsCache { - private final HashMap> positions = new HashMap<>(); - - public boolean contains(@NonNull Language language) { - return positions.containsKey(language.getId()); - } - - public void put(@NonNull Language language, @NonNull String sequence, int wordCount) { - if (wordCount < SettingsStore.SUGGESTIONS_POSITIONS_LIMIT && !contains(language)) { - positions.put(language.getId(), null); - return; - } - - HashMap words = positions.get(language.getId()); - if (words == null) { - words = new HashMap<>(); - positions.put(language.getId(), words); - } - words.put(sequence, wordCount); - } - - public int get(@NonNull Language language, @NonNull String sequence) { - if (!contains(language)) { - return SettingsStore.SUGGESTIONS_POSITIONS_LIMIT; - } - - HashMap words = positions.get(language.getId()); - if (words == null) { - return SettingsStore.SUGGESTIONS_POSITIONS_LIMIT; - } - - Integer wordCount = words.get(sequence); - return wordCount == null ? SettingsStore.SUGGESTIONS_POSITIONS_LIMIT : wordCount; - } -} diff --git a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/InsertOps.java b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/InsertOps.java index b4a29919..3a2d0690 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/InsertOps.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/InsertOps.java @@ -42,10 +42,11 @@ public class InsertOps { } - public static void replaceLanguageMeta(@NonNull SQLiteDatabase db, int langId, String fileHash) { - SQLiteStatement query = CompiledQueryCache.get(db, "REPLACE INTO " + Tables.LANGUAGES_META + " (langId, fileHash) VALUES (?, ?)"); + public static void replaceLanguageMeta(@NonNull SQLiteDatabase db, int langId, String fileHash, int maxWordsPerSequence) { + SQLiteStatement query = CompiledQueryCache.get(db, "REPLACE INTO " + Tables.LANGUAGES_META + " (langId, fileHash, maxWordsPerSequence) VALUES (?, ?, ?)"); query.bindLong(1, langId); query.bindString(2, fileHash); + query.bindLong(3, maxWordsPerSequence); query.execute(); } diff --git a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Migration.java b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Migration.java index bb53014c..ab215876 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Migration.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Migration.java @@ -13,6 +13,10 @@ class Migration { new Migration( "ALTER TABLE " + Tables.LANGUAGES_META + " ADD COLUMN positionsToNormalize TEXT NULL", true + ), + new Migration( + "ALTER TABLE " + Tables.LANGUAGES_META + " ADD COLUMN maxWordsPerSequence INTEGER NOT NULL DEFAULT -1", + true ) }; diff --git a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/ReadOps.java b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/ReadOps.java index de39ead7..de9c13b5 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/ReadOps.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/ReadOps.java @@ -11,8 +11,8 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; import java.util.ArrayList; +import java.util.HashMap; -import io.github.sspanak.tt9.db.entities.LongPositionsCache; import io.github.sspanak.tt9.db.entities.NormalizationList; import io.github.sspanak.tt9.db.entities.WordList; import io.github.sspanak.tt9.db.entities.WordPositionsStringBuilder; @@ -26,7 +26,7 @@ import io.github.sspanak.tt9.util.Logger; public class ReadOps { private final String LOG_TAG = "ReadOperations"; - private final LongPositionsCache longPositionsCache = new LongPositionsCache(); + private final HashMap maxWordsPerSequence = new HashMap<>(); /** @@ -256,7 +256,7 @@ public class ReadOps { .append(sequence) .append("' OR sequence BETWEEN '").append(sequence).append("1' AND '").append(sequence).append(rangeEnd).append("'"); sql.append(" ORDER BY `start` "); - sql.append(" LIMIT ").append(longPositionsCache.get(language, sequence)); + sql.append(" LIMIT ").append(maxWordsPerSequence.get(language)); } String positionsSql = sql.toString(); @@ -342,28 +342,17 @@ public class ReadOps { /** - * Returns the sequences that result in more words than the standard performance-balanced limit of 100. + * Caches the languages with more than 100 words per a sequence (the balanced performance limit). */ public void cacheLongPositionsIfMissing(@NonNull SQLiteDatabase db, @NonNull Language language) { - if (longPositionsCache.contains(language)) { + if (maxWordsPerSequence.containsKey(language)) { return; } - String[] select = new String[]{"sequence", "`end` - `start`"}; - String table = Tables.getWordPositions(language.getId()); - String where = "LENGTH(sequence) > " + SettingsStore.SUGGESTIONS_POSITIONS_LIMIT; + String sql = "SELECT maxWordsPerSequence FROM " + Tables.LANGUAGES_META + " WHERE langId = " + language.getId(); + int maxWords = (int) CompiledQueryCache.simpleQueryForLong(db, sql, SettingsStore.SUGGESTIONS_POSITIONS_LIMIT); + maxWords = maxWords > 0 ? maxWords : SettingsStore.SUGGESTIONS_POSITIONS_LIMIT; - boolean hasResults = false; - - try (Cursor cursor = db.query(table, select, where, null, null, null, null)) { - while (cursor.moveToNext()) { - hasResults = true; - longPositionsCache.put(language, cursor.getString(0), cursor.getInt(1)); - } - } - - if (!hasResults) { - longPositionsCache.put(language, "", 0); - } + maxWordsPerSequence.put(language, maxWords); } } diff --git a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Tables.java b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Tables.java index b71da3e5..92ab8cd4 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Tables.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Tables.java @@ -115,7 +115,8 @@ public class Tables { return "CREATE TABLE IF NOT EXISTS " + LANGUAGES_META + " (" + "langId INTEGER UNIQUE NOT NULL, " + "positionsToNormalize TEXT NULL," + - "fileHash TEXT NOT NULL DEFAULT 0 " + + "fileHash TEXT NOT NULL DEFAULT 0, " + + "maxWordsPerSequence INTEGER NOT NULL DEFAULT -1 " + ")"; } } diff --git a/app/src/main/java/io/github/sspanak/tt9/db/words/DictionaryLoader.java b/app/src/main/java/io/github/sspanak/tt9/db/words/DictionaryLoader.java index a83c8d31..2a20bf8c 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/words/DictionaryLoader.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/words/DictionaryLoader.java @@ -263,6 +263,7 @@ public class DictionaryLoader { WordBatch batch = new WordBatch(language, SettingsStore.DICTIONARY_IMPORT_BATCH_SIZE + 1); float progressRatio = (maxProgress - minProgress) / wordFile.getWords(); int wordCount = 0; + int maxWordCount = 0; try (BufferedReader ignored = wordFile.getReader()) { while (wordFile.notEOF()) { @@ -276,6 +277,7 @@ public class DictionaryLoader { ArrayList words = wordFile.getNextWords(digitSequence); batch.add(words, digitSequence, wordCount + positionShift); wordCount += words.size(); + maxWordCount = Math.max(maxWordCount, wordCount); if (batch.getWords().size() > SettingsStore.DICTIONARY_IMPORT_BATCH_SIZE) { saveWordBatch(batch); @@ -290,7 +292,7 @@ public class DictionaryLoader { } saveWordBatch(batch); - InsertOps.replaceLanguageMeta(sqlite.getDb(), language.getId(), wordFile.getHash()); + InsertOps.replaceLanguageMeta(sqlite.getDb(), language.getId(), wordFile.getHash(), maxWordCount); }