wzray/tt9
1
0
Fork 0
Code Activity

fixed a crash when loading very large dictionaries on low memory devices

Browse source
This commit is contained in:
sspanak 2024-02-09 23:18:57 +02:00 • committed by Dimo Karaivanov
parent 5cdf979836
commit ec87ce080c
3 changed files with 33 additions and 70 deletions
Download patch file Download diff file Expand all files Collapse all files

88
src/io/github/sspanak/tt9/db/DictionaryLoader.java
View file

@ -37,8 +37,8 @@ public class DictionaryLoader {
private ConsumerCompat<Bundle> onStatusChange; private ConsumerCompat<Bundle> onStatusChange;
private Thread loadThread; private Thread loadThread;
private long importStartTime = 0;
private int currentFile = 0; private int currentFile = 0;
private long importStartTime = 0;
private long lastProgressUpdate = 0; private long lastProgressUpdate = 0;
@ -121,7 +121,6 @@ public class DictionaryLoader {
try { try {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
float progress = 1; float progress = 1;
final float dictionaryMaxProgress = 90f;
sqlite.beginTransaction(); sqlite.beginTransaction();
@ -145,16 +144,10 @@ public class DictionaryLoader {
logLoadingStep("Custom words restored", language, start); logLoadingStep("Custom words restored", language, start);
start = System.currentTimeMillis(); start = System.currentTimeMillis();
WordBatch words = readWordsFile(language, lettersCount, progress, progress + 25f); importWordFile(language, lettersCount, progress, 90);
progress += 25; progress = 90;
sendProgressMessage(language, progress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME); sendProgressMessage(language, progress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
logLoadingStep("Dictionary file loaded in memory", language, start); logLoadingStep("Dictionary file imported", language, start);
start = System.currentTimeMillis();
saveWordBatch(words, progress, dictionaryMaxProgress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
progress = dictionaryMaxProgress;
sendProgressMessage(language, progress, 0);
logLoadingStep("Dictionary words saved in database", language, start);
start = System.currentTimeMillis(); start = System.currentTimeMillis();
Tables.createPositionIndex(sqlite.getDb(), language); Tables.createPositionIndex(sqlite.getDb(), language);
@ -197,7 +190,7 @@ public class DictionaryLoader {
} }
private int importLetters(Language language) throws InvalidLanguageCharactersException, DictionaryImportAbortedException { private int importLetters(Language language) throws InvalidLanguageCharactersException {
int lettersCount = 0; int lettersCount = 0;
boolean isEnglish = language.getLocale().equals(Locale.ENGLISH); boolean isEnglish = language.getLocale().equals(Locale.ENGLISH);
WordBatch letters = new WordBatch(language); WordBatch letters = new WordBatch(language);
@ -210,13 +203,13 @@ public class DictionaryLoader {
} }
} }
saveWordBatch(letters, -1, -1, -1); saveWordBatch(letters);
return lettersCount; return lettersCount;
} }
private WordBatch readWordsFile(Language language, int positionShift, float minProgress, float maxProgress) throws Exception { private void importWordFile(Language language, int positionShift, float minProgress, float maxProgress) throws Exception {
int currentLine = 1; int currentLine = 1;
int totalLines = getFileSize(language.getDictionaryFile()); int totalLines = getFileSize(language.getDictionaryFile());
float progressRatio = (maxProgress - minProgress) / totalLines; float progressRatio = (maxProgress - minProgress) / totalLines;
@ -235,74 +228,35 @@ public class DictionaryLoader {
short frequency = getFrequency(parts); short frequency = getFrequency(parts);
try { try {
batch.add(word, frequency, currentLine + positionShift); boolean isFinalized = batch.add(word, frequency, currentLine + positionShift);
if (isFinalized && batch.getWords().size() > SettingsStore.DICTIONARY_IMPORT_BATCH_SIZE) {
saveWordBatch(batch);
batch.clear();
}
} catch (InvalidLanguageCharactersException e) { } catch (InvalidLanguageCharactersException e) {
throw new DictionaryImportException(word, currentLine); throw new DictionaryImportException(word, currentLine);
} }
if (totalLines > 0 && currentLine % SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_BATCH_SIZE == 0) { if (totalLines > 0) {
sendProgressMessage(language, minProgress + progressRatio * currentLine, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME); sendProgressMessage(language, minProgress + progressRatio * currentLine, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
} }
} }
} }
return batch; saveWordBatch(batch);
InsertOps.insertLanguageMeta(sqlite.getDb(), language.getId());
} }
public void saveWordBatch(WordBatch batch, float minProgress, float maxProgress, int sizeUpdateInterval) throws DictionaryImportAbortedException { public void saveWordBatch(WordBatch batch) {
float middleProgress = minProgress + (maxProgress - minProgress) / 2;
InsertOps insertOps = new InsertOps(sqlite.getDb(), batch.getLanguage()); InsertOps insertOps = new InsertOps(sqlite.getDb(), batch.getLanguage());
insertWordsBatch(insertOps, batch, minProgress, middleProgress - 2, sizeUpdateInterval); for (int i = 0, end = batch.getWords().size(); i < end; i++) {
insertWordPositionsBatch(insertOps, batch, middleProgress - 2, maxProgress - 2, sizeUpdateInterval); insertOps.insertWord(batch.getWords().get(i));
InsertOps.insertLanguageMeta(sqlite.getDb(), batch.getLanguage().getId());
if (sizeUpdateInterval > 0) {
sendProgressMessage(batch.getLanguage(), maxProgress, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
}
}
private void insertWordsBatch(InsertOps insertOps, WordBatch batch, float minProgress, float maxProgress, int sizeUpdateInterval) throws DictionaryImportAbortedException {
if (batch.getWords().size() == 0) {
return;
} }
float progressRatio = (maxProgress - minProgress) / batch.getWords().size(); for (int i = 0, end = batch.getPositions().size(); i < end; i++) {
insertOps.insertWordPosition(batch.getPositions().get(i));
for (int progress = 0, end = batch.getWords().size(); progress < end; progress++) {
if (loadThread.isInterrupted()) {
sendProgressMessage(batch.getLanguage(), 0, 0);
throw new DictionaryImportAbortedException();
}
insertOps.insertWord(batch.getWords().get(progress));
if (sizeUpdateInterval > 0 && progress % sizeUpdateInterval == 0) {
sendProgressMessage(batch.getLanguage(), minProgress + progress * progressRatio, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
}
}
}
private void insertWordPositionsBatch(InsertOps insertOps, WordBatch batch, float minProgress, float maxProgress, int sizeUpdateInterval) throws DictionaryImportAbortedException {
if (batch.getPositions().size() == 0) {
return;
}
float progressRatio = (maxProgress - minProgress) / batch.getPositions().size();
for (int progress = 0, end = batch.getPositions().size(); progress < end; progress++) {
if (loadThread.isInterrupted()) {
sendProgressMessage(batch.getLanguage(), 0, 0);
throw new DictionaryImportAbortedException();
}
insertOps.insertWordPosition(batch.getPositions().get(progress));
if (sizeUpdateInterval > 0 && progress % sizeUpdateInterval == 0) {
sendProgressMessage(batch.getLanguage(), minProgress + progress * progressRatio, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME);
}
} }
} }
@ -374,7 +328,7 @@ public class DictionaryLoader {
Bundle progressMsg = new Bundle(); Bundle progressMsg = new Bundle();
progressMsg.putInt("languageId", language.getId()); progressMsg.putInt("languageId", language.getId());
progressMsg.putLong("time", getImportTime()); progressMsg.putLong("time", getImportTime());
progressMsg.putInt("progress", (int) Math.round(progress)); progressMsg.putInt("progress", Math.round(progress));
progressMsg.putInt("currentFile", currentFile); progressMsg.putInt("currentFile", currentFile);
asyncHandler.post(() -> onStatusChange.accept(progressMsg)); asyncHandler.post(() -> onStatusChange.accept(progressMsg));
} }

13
src/io/github/sspanak/tt9/db/entities/WordBatch.java
View file

@ -24,11 +24,11 @@ public class WordBatch {
this(language, 0); this(language, 0);
} }
public void add(@NonNull String word, int frequency, int position) throws InvalidLanguageCharactersException { public boolean add(@NonNull String word, int frequency, int position) throws InvalidLanguageCharactersException {
words.add(Word.create(word, frequency, position)); words.add(Word.create(word, frequency, position));
if (position == 0) { if (position == 0) {
return; return true;
} }
String sequence = language.getDigitSequenceForWord(word); String sequence = language.getDigitSequenceForWord(word);
@ -44,7 +44,16 @@ public class WordBatch {
positions.add(lastWordPosition); positions.add(lastWordPosition);
lastWordPosition = WordPosition.create(sequence, position); lastWordPosition = WordPosition.create(sequence, position);
return true;
} }
return false;
}
public void clear() {
words.clear();
positions.clear();
} }
@NonNull public Language getLanguage() { @NonNull public Language getLanguage() {

2
src/io/github/sspanak/tt9/preferences/SettingsStore.java
View file

@ -272,7 +272,7 @@ public class SettingsStore {
public boolean getDebugLogsEnabled() { return prefs.getBoolean("pref_enable_debug_logs", Logger.isDebugLevel()); } public boolean getDebugLogsEnabled() { return prefs.getBoolean("pref_enable_debug_logs", Logger.isDebugLevel()); }
public final static int DICTIONARY_IMPORT_PROGRESS_UPDATE_BATCH_SIZE = 1000; // items public final static int DICTIONARY_IMPORT_BATCH_SIZE = 5000; // words
public final static int DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME = 250; // ms public final static int DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME = 250; // ms
public final static int DICTIONARY_MISSING_WARNING_INTERVAL = 30000; // ms public final static int DICTIONARY_MISSING_WARNING_INTERVAL = 30000; // ms
public final static int PREFERENCES_CLICK_DEBOUNCE_TIME = 250; // ms public final static int PREFERENCES_CLICK_DEBOUNCE_TIME = 250; // ms