From 17b8a30f44eb9b50679e44e0a347109dc6000aec Mon Sep 17 00:00:00 2001 From: sspanak Date: Fri, 17 May 2024 20:29:29 +0300 Subject: [PATCH] simplified normalization algorithm: now only the sequnce over the top is normalized, not the entire language --- .../io/github/sspanak/tt9/db/WordStore.java | 15 ++++---- .../tt9/db/entities/NormalizationList.java | 35 +++++++++++++++++++ .../tt9/db/sqlite/CompiledQueryCache.java | 8 +++++ .../sspanak/tt9/db/sqlite/Migration.java | 8 +++++ .../github/sspanak/tt9/db/sqlite/ReadOps.java | 11 +++--- .../github/sspanak/tt9/db/sqlite/Tables.java | 2 +- .../sspanak/tt9/db/sqlite/UpdateOps.java | 24 +++++++------ .../preferences/settings/SettingsStore.java | 2 +- 8 files changed, 81 insertions(+), 24 deletions(-) create mode 100644 app/src/main/java/io/github/sspanak/tt9/db/entities/NormalizationList.java diff --git a/app/src/main/java/io/github/sspanak/tt9/db/WordStore.java b/app/src/main/java/io/github/sspanak/tt9/db/WordStore.java index 99fd51b4..acf5e231 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/WordStore.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/WordStore.java @@ -6,6 +6,7 @@ import androidx.annotation.NonNull; import java.util.ArrayList; +import io.github.sspanak.tt9.db.entities.NormalizationList; import io.github.sspanak.tt9.db.entities.Word; import io.github.sspanak.tt9.db.entities.WordList; import io.github.sspanak.tt9.db.sqlite.DeleteOps; @@ -224,7 +225,7 @@ public class WordStore { } if (newTopFrequency > SettingsStore.WORD_FREQUENCY_MAX) { - scheduleNormalization(language); + scheduleNormalization(language, topWordPositions); } Logger.d(LOG_TAG, "Changed frequency of '" + word + "' to: " + newTopFrequency + ". Time: " + Timer.stop(LOG_TAG) + " ms"); @@ -243,11 +244,11 @@ public class WordStore { try { sqlite.beginTransaction(); - int nextLangId = readOps.getNextInNormalizationQueue(sqlite.getDb()); - UpdateOps.normalize(sqlite.getDb(), nextLangId); + NormalizationList normalizationList = readOps.getNextInNormalizationQueue(sqlite.getDb()); + UpdateOps.normalize(sqlite.getDb(), normalizationList); sqlite.finishTransaction(); - String message = nextLangId > 0 ? "Normalized language: " + nextLangId : "No languages to normalize"; + String message = normalizationList.langId > 0 ? "Normalized language: " + normalizationList.langId + ", positions: " + normalizationList.positions : "No languages to normalize"; Logger.d(LOG_TAG, message + ". Time: " + Timer.stop(LOG_TAG) + " ms"); } catch (Exception e) { sqlite.failTransaction(); @@ -256,9 +257,9 @@ public class WordStore { } - public void scheduleNormalization(Language language) { - if (language != null && !(language instanceof NullLanguage) && checkOrNotify()) { - UpdateOps.scheduleNormalization(sqlite.getDb(), language); + public void scheduleNormalization(Language language, String positions) { + if (language != null && !(language instanceof NullLanguage) && positions != null && !positions.isEmpty() && checkOrNotify()) { + UpdateOps.scheduleNormalization(sqlite.getDb(), language, positions); } } diff --git a/app/src/main/java/io/github/sspanak/tt9/db/entities/NormalizationList.java b/app/src/main/java/io/github/sspanak/tt9/db/entities/NormalizationList.java new file mode 100644 index 00000000..43de3d95 --- /dev/null +++ b/app/src/main/java/io/github/sspanak/tt9/db/entities/NormalizationList.java @@ -0,0 +1,35 @@ +package io.github.sspanak.tt9.db.entities; + +import java.util.regex.Pattern; + +public class NormalizationList { + private final static Pattern validPositions = Pattern.compile("^[0-9,]+\\d|\\d+$"); + + public int langId = -1; + public String positions = null; + + public NormalizationList(String rawNormalizationResponse) { + if (rawNormalizationResponse == null) { + return; + } + + String[] parts = rawNormalizationResponse.split(",", 2); + if (arePartsValid(parts)) { + langId = Integer.parseInt(parts[0]); + positions = parts[1]; + } + } + + private boolean arePartsValid(String[] parts) { + if (parts.length != 2 || !validPositions.matcher(parts[1]).matches()) { + return false; + } + + try { + Integer.parseInt(parts[0]); + return true; + } catch (Exception e) { + return false; + } + } +} diff --git a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/CompiledQueryCache.java b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/CompiledQueryCache.java index b27afb02..1be046e6 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/CompiledQueryCache.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/CompiledQueryCache.java @@ -60,4 +60,12 @@ class CompiledQueryCache { static long simpleQueryForLong(SQLiteDatabase db, String sql, long defaultValue) { return getInstance(db).simpleQueryForLong(sql, defaultValue); } + + static String simpleQueryForString(SQLiteDatabase db, String sql, String defaultValue) { + try { + return get(db, sql).simpleQueryForString(); + } catch (SQLiteDoneException e) { + return defaultValue; + } + } } diff --git a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Migration.java b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Migration.java index 7376f661..bb53014c 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Migration.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Migration.java @@ -5,6 +5,14 @@ class Migration { new Migration( "ALTER TABLE " + Tables.LANGUAGES_META + " ADD COLUMN fileHash TEXT NOT NULL DEFAULT 0", true + ), + new Migration( + "ALTER TABLE " + Tables.LANGUAGES_META + " RENAME COLUMN normalizationPending TO _delete_me_0", + true + ), + new Migration( + "ALTER TABLE " + Tables.LANGUAGES_META + " ADD COLUMN positionsToNormalize TEXT NULL", + true ) }; diff --git a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/ReadOps.java b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/ReadOps.java index d450d247..59559286 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/ReadOps.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/ReadOps.java @@ -9,6 +9,7 @@ import androidx.annotation.NonNull; import java.util.ArrayList; +import io.github.sspanak.tt9.db.entities.NormalizationList; import io.github.sspanak.tt9.util.Logger; import io.github.sspanak.tt9.db.SlowQueryStats; import io.github.sspanak.tt9.db.entities.WordList; @@ -274,11 +275,13 @@ public class ReadOps { } - public int getNextInNormalizationQueue(@NonNull SQLiteDatabase db) { - return (int) CompiledQueryCache.simpleQueryForLong( + public NormalizationList getNextInNormalizationQueue(@NonNull SQLiteDatabase db) { + String res = CompiledQueryCache.simpleQueryForString( db, - "SELECT langId FROM " + Tables.LANGUAGES_META + " WHERE normalizationPending = 1 LIMIT 1", - -1 + "SELECT langId || ',' || positionsToNormalize FROM " + Tables.LANGUAGES_META + " WHERE positionsToNormalize IS NOT NULL LIMIT 1", + null ); + + return new NormalizationList(res); } } diff --git a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Tables.java b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Tables.java index 6d8319f0..5acd66a9 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Tables.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/Tables.java @@ -103,7 +103,7 @@ public class Tables { private static String createLanguagesMeta() { return "CREATE TABLE IF NOT EXISTS " + LANGUAGES_META + " (" + "langId INTEGER UNIQUE NOT NULL, " + - "normalizationPending INT2 NOT NULL DEFAULT 0," + + "positionsToNormalize TEXT NULL," + "fileHash TEXT NOT NULL DEFAULT 0 " + ")"; } diff --git a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/UpdateOps.java b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/UpdateOps.java index b84e9966..1002edb0 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/sqlite/UpdateOps.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/sqlite/UpdateOps.java @@ -5,6 +5,7 @@ import android.database.sqlite.SQLiteStatement; import androidx.annotation.NonNull; +import io.github.sspanak.tt9.db.entities.NormalizationList; import io.github.sspanak.tt9.languages.Language; import io.github.sspanak.tt9.preferences.settings.SettingsStore; import io.github.sspanak.tt9.util.Logger; @@ -37,25 +38,26 @@ public class UpdateOps { } - public static void normalize(@NonNull SQLiteDatabase db, int langId) { - if (langId <= 0) { + public static void normalize(@NonNull SQLiteDatabase db, NormalizationList normalizationList) { + if (normalizationList.langId <= 0 || normalizationList.positions == null || normalizationList.positions.isEmpty()) { return; } - SQLiteStatement query = CompiledQueryCache.get(db, "UPDATE " + Tables.getWords(langId) + " SET frequency = frequency / ?"); - query.bindLong(1, SettingsStore.WORD_FREQUENCY_NORMALIZATION_DIVIDER); - query.execute(); + db.execSQL( + "UPDATE " + Tables.getWords(normalizationList.langId) + + " SET frequency = frequency / " + SettingsStore.WORD_FREQUENCY_NORMALIZATION_DIVIDER + + " WHERE position IN (" + normalizationList.positions + ")" + ); - query = CompiledQueryCache.get(db, "UPDATE " + Tables.LANGUAGES_META + " SET normalizationPending = ? WHERE langId = ?"); - query.bindLong(1, 0); - query.bindLong(2, langId); + SQLiteStatement query = CompiledQueryCache.get(db, "UPDATE " + Tables.LANGUAGES_META + " SET positionsToNormalize = NULL WHERE langId = ?"); + query.bindLong(1, normalizationList.langId); query.execute(); } - public static void scheduleNormalization(@NonNull SQLiteDatabase db, @NonNull Language language) { - SQLiteStatement query = CompiledQueryCache.get(db, "UPDATE " + Tables.LANGUAGES_META + " SET normalizationPending = ? WHERE langId = ?"); - query.bindLong(1, 1); + public static void scheduleNormalization(@NonNull SQLiteDatabase db, @NonNull Language language, @NonNull String positions) { + SQLiteStatement query = CompiledQueryCache.get(db, "UPDATE " + Tables.LANGUAGES_META + " SET positionsToNormalize = ? WHERE langId = ?"); + query.bindString(1, positions); query.bindLong(2, language.getId()); query.execute(); } diff --git a/app/src/main/java/io/github/sspanak/tt9/preferences/settings/SettingsStore.java b/app/src/main/java/io/github/sspanak/tt9/preferences/settings/SettingsStore.java index 3420282a..10393beb 100644 --- a/app/src/main/java/io/github/sspanak/tt9/preferences/settings/SettingsStore.java +++ b/app/src/main/java/io/github/sspanak/tt9/preferences/settings/SettingsStore.java @@ -22,7 +22,7 @@ public class SettingsStore extends SettingsUI { public final static int SUGGESTIONS_TRANSLATE_ANIMATION_DURATION = 0; public final static int WORD_FREQUENCY_MAX = 25500; public final static int WORD_FREQUENCY_NORMALIZATION_DIVIDER = 100; // normalized frequency = WORD_FREQUENCY_MAX / WORD_FREQUENCY_NORMALIZATION_DIVIDER - public final static int WORD_NORMALIZATION_DELAY = 120000; // ms + public final static int WORD_NORMALIZATION_DELAY = 15000; // ms /************* hacks *************/ public final static int PREFERENCES_CLICK_DEBOUNCE_TIME = 250; // ms