New dictionary loader (#89)

* new, simpler (and hopefully, more efficient) dictionary loader

* no more dict.properties

* dictionaries are now validated during the build process

* TraditionalT9Settings code cleanup and code style improvements

* removed English, French, Italian, Russian repeating words

* removed invalid and repeating German words

src/io/github/sspanak/tt9/db/DictionaryDb.java

@@ -59,16 +59,8 @@ public class DictionaryDb {
 	}
 
 
-	public static void beginTransaction() {
-		getInstance().beginTransaction();
-	}
-
-
-	public static void endTransaction(boolean success) {
-		if (success) {
-			getInstance().setTransactionSuccessful();
-		}
-		getInstance().endTransaction();
+	public static void runInTransaction(Runnable r) {
+		getInstance().runInTransaction(r);
 	}
 
 

src/io/github/sspanak/tt9/db/DictionaryImportAbortedException.java

@@ -0,0 +1,7 @@
+package io.github.sspanak.tt9.db;
+
+public class DictionaryImportAbortedException extends Exception{
+	public DictionaryImportAbortedException() {
+		super("Dictionary import stopped by request.");
+	}
+}

src/io/github/sspanak/tt9/db/DictionaryImportException.java

@@ -0,0 +1,14 @@
+package io.github.sspanak.tt9.db;
+
+public class DictionaryImportException extends Exception {
+	public final String file;
+	public final String word;
+	public final long line;
+
+	DictionaryImportException(String file, String word, long line) {
+		super("Dictionary import failed");
+		this.file = file;
+		this.word = word;
+		this.line = line;
+	}
+}

src/io/github/sspanak/tt9/db/DictionaryLoader.java

@@ -0,0 +1,259 @@
+package io.github.sspanak.tt9.db;
+
+import android.content.Context;
+import android.content.res.AssetManager;
+import android.os.Bundle;
+import android.os.Handler;
+import android.os.Message;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.LineNumberReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.regex.Pattern;
+
+import io.github.sspanak.tt9.Logger;
+import io.github.sspanak.tt9.languages.InvalidLanguageCharactersException;
+import io.github.sspanak.tt9.languages.InvalidLanguageException;
+import io.github.sspanak.tt9.languages.Language;
+import io.github.sspanak.tt9.preferences.T9Preferences;
+
+public class DictionaryLoader {
+	private final AssetManager assets;
+	private final T9Preferences prefs;
+
+	private boolean isStopped = true;
+	private int currentFile = 0;
+	private long lastProgressUpdate = 0;
+
+	private final Pattern containsPunctuation = Pattern.compile("\\p{Punct}(?<!-)");
+
+	public DictionaryLoader(Context context) {
+		assets = context.getAssets();
+		prefs = T9Preferences.getInstance();
+	}
+
+
+	public void load(Handler handler, ArrayList<Language> languages) {
+		new Thread() {
+			@Override
+			public void run() {
+				currentFile = 0;
+				isStopped = false;
+				// SQLite does not support parallel queries, so let's import them one by one
+				for (Language lang : languages) {
+					if (isStopped) {
+						break;
+					}
+					importAll(handler, lang);
+					currentFile++;
+				}
+			}
+		}.start();
+	}
+
+
+	public void stop() {
+		isStopped = true;
+	}
+
+
+	private void importAll(Handler handler, Language language) {
+		final String logTag = "tt9.DictionaryLoader.importAll";
+
+		if (language == null) {
+			Logger.e(logTag, "Failed loading a dictionary for NULL language.");
+			sendError(handler, InvalidLanguageException.class.getSimpleName(), -1);
+			return;
+		}
+
+		DictionaryDb.runInTransaction(() -> {
+			long start = System.currentTimeMillis();
+			importLetters(language);
+			Logger.i(
+				logTag,
+				"Loaded letters for '" + language.getName() + "' language in: " + (System.currentTimeMillis() - start) + " ms"
+			);
+
+			try {
+				start = System.currentTimeMillis();
+				importWords(handler, language);
+				Logger.i(
+					logTag,
+					"Dictionary: '" + language.getDictionaryFile() + "'" +
+						" processing time: " + (System.currentTimeMillis() - start) + " ms"
+				);
+			} catch (DictionaryImportAbortedException e) {
+				stop();
+
+				Logger.i(
+					logTag,
+					e.getMessage() + ". File '" + language.getDictionaryFile() + "' not imported."
+				);
+			} catch (DictionaryImportException e) {
+				stop();
+				sendImportError(handler, DictionaryImportException.class.getSimpleName(), language.getId(), e.line, e.word);
+
+				Logger.e(
+					logTag,
+					" Invalid word: '" + e.word
+					+ "' in dictionary: '" + language.getDictionaryFile() + "'"
+					+ " on line " + e.line
+					+ " of language '" + language.getName() + "'. "
+					+ e.getMessage()
+				);
+			} catch (Exception e) {
+				stop();
+				sendError(handler, e.getClass().getSimpleName(), language.getId());
+
+				Logger.e(
+					logTag,
+					"Failed loading dictionary: " + language.getDictionaryFile() +
+					" for language '" + language.getName() + "'. "
+					+ e.getMessage()
+				);
+			}
+		});
+	}
+
+
+	private void importLetters(Language language) {
+		ArrayList<Word> letters = new ArrayList<>();
+
+		for (int key = 0; key <= 9; key++) {
+			for (String langChar : language.getKeyCharacters(key)) {
+				if (langChar.length() == 1 && langChar.charAt(0) >= '0' && langChar.charAt(0) <= '9') {
+					// We do not want 0-9 as "word suggestions" in Predictive mode. It looks confusing
+					// when trying to type a word and also, one can type them by holding the respective
+					// key.
+					continue;
+				}
+
+				Word word = new Word();
+				word.langId = language.getId();
+				word.frequency = 0;
+				word.sequence = String.valueOf(key);
+				word.word = langChar;
+
+				letters.add(word);
+			}
+		}
+
+		DictionaryDb.insertWordsSync(letters);
+	}
+
+
+	private void importWords(Handler handler, Language language) throws Exception {
+		importWords(handler, language, language.getDictionaryFile());
+	}
+
+
+	private void importWords(Handler handler, Language language, String dictionaryFile) throws Exception {
+		long totalWords = countWords(dictionaryFile);
+
+		BufferedReader br = new BufferedReader(new InputStreamReader(assets.open(dictionaryFile), StandardCharsets.UTF_8));
+
+		ArrayList<Word> dbWords = new ArrayList<>();
+		long line = 0;
+
+		sendProgressMessage(handler, language, 0, 0);
+
+		for (String word; (word = br.readLine()) != null; line++) {
+			if (isStopped) {
+				br.close();
+				sendProgressMessage(handler, language, 0, 0);
+				throw new DictionaryImportAbortedException();
+			}
+
+			validateWord(language, word, line);
+			dbWords.add(stringToWord(language, word));
+
+			if (line % prefs.getDictionaryImportWordChunkSize() == 0) {
+				DictionaryDb.insertWordsSync(dbWords);
+				dbWords.clear();
+			}
+
+			if (totalWords > 0) {
+				int progress = (int) Math.floor(100.0 * line / totalWords);
+				sendProgressMessage(handler, language, progress, prefs.getDictionaryImportProgressUpdateInterval());
+			}
+		}
+
+		br.close();
+		sendProgressMessage(handler, language, 100, 0);
+	}
+
+
+	private long countWords(String filename) {
+		try (LineNumberReader reader = new LineNumberReader(new InputStreamReader(assets.open(filename), StandardCharsets.UTF_8))) {
+			//noinspection ResultOfMethodCallIgnored
+			reader.skip(Long.MAX_VALUE);
+			long lines = reader.getLineNumber();
+			reader.close();
+
+			return lines;
+		} catch (Exception e) {
+			Logger.w("DictionaryLoader.countWords", "Could not count the lines of file: " + filename + ". " + e.getMessage());
+			return 0;
+		}
+	}
+
+
+	private void validateWord(Language language, String word, long line) throws DictionaryImportException {
+		if (!language.isPunctuationPartOfWords() && containsPunctuation.matcher(word).find()) {
+			throw new DictionaryImportException(language.getDictionaryFile(), word, line);
+		}
+	}
+
+
+	private Word stringToWord(Language language, String word) throws InvalidLanguageCharactersException {
+		Word dbWord = new Word();
+		dbWord.langId = language.getId();
+		dbWord.frequency = 0;
+		dbWord.sequence = language.getDigitSequenceForWord(word);
+		dbWord.word = word;
+
+		return dbWord;
+	}
+
+
+	private void sendProgressMessage(Handler handler, Language language, int progress, int progressUpdateInterval) {
+		long now = System.currentTimeMillis();
+		if (now - lastProgressUpdate < progressUpdateInterval) {
+			return;
+		}
+
+		lastProgressUpdate = now;
+
+		Bundle bundle = new Bundle();
+		bundle.putInt("languageId", language.getId());
+		bundle.putInt("progress", progress);
+		bundle.putInt("currentFile", currentFile);
+		Message msg = new Message();
+		msg.setData(bundle);
+		handler.sendMessage(msg);
+	}
+
+
+	private void sendError(Handler handler, String message, int langId) {
+		Bundle bundle = new Bundle();
+		bundle.putString("error", message);
+		bundle.putInt("languageId", langId);
+		Message msg = new Message();
+		msg.setData(bundle);
+		handler.sendMessage(msg);
+	}
+
+
+	private void sendImportError(Handler handler, String message, int langId, long fileLine, String word) {
+		Bundle bundle = new Bundle();
+		bundle.putString("error", message);
+		bundle.putLong("fileLine", fileLine);
+		bundle.putInt("languageId", langId);
+		bundle.putString("word", word);
+		Message msg = new Message();
+		msg.setData(bundle);
+		handler.sendMessage(msg);
+	}
+}

src/io/github/sspanak/tt9/languages/InvalidLanguageCharactersException.java

@@ -0,0 +1,15 @@
+package io.github.sspanak.tt9.languages;
+
+public class InvalidLanguageCharactersException extends Exception {
+	private Language language;
+
+	public InvalidLanguageCharactersException(Language language, String extraMessage) {
+		super("Some characters are not supported in language: " + language.getName() + ". " + extraMessage);
+		this.language = language;
+	}
+
+	public Language getLanguage() {
+		return language;
+	}
+}
+

src/io/github/sspanak/tt9/languages/Language.java

@@ -70,7 +70,7 @@ public class Language {
 		return chars;
 	}
 
-	public String getDigitSequenceForWord(String word) throws Exception {
+	public String getDigitSequenceForWord(String word) throws InvalidLanguageCharactersException {
 		StringBuilder sequence = new StringBuilder();
 		String lowerCaseWord = word.toLowerCase(locale);
 
@@ -83,9 +83,7 @@ public class Language {
 		}
 
 		if (word.length() != sequence.length()) {
-			throw new Exception(
-				"Failed generating digit sequence for word: '" + word + "'. Some characters are not supported in language: " + name
-			);
+			throw new InvalidLanguageCharactersException(this, "Failed generating digit sequence for word: '" + word);
 		}
 
 		return sequence.toString();

src/io/github/sspanak/tt9/preferences/T9Preferences.java

@@ -2,10 +2,10 @@ package io.github.sspanak.tt9.preferences;
 
 import android.content.Context;
 import android.content.SharedPreferences;
-import androidx.preference.PreferenceManager;
-
 import android.view.KeyEvent;
 
+import androidx.preference.PreferenceManager;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 
@@ -23,11 +23,13 @@ public class T9Preferences {
 	private final SharedPreferences prefs;
 	private final SharedPreferences.Editor prefsEditor;
 
+
 	public T9Preferences (Context context) {
 		prefs = PreferenceManager.getDefaultSharedPreferences(context);
 		prefsEditor = prefs.edit();
 	}
 
+
 	public static T9Preferences getInstance() {
 		if (self == null) {
 			self = new T9Preferences(TraditionalT9.getMainContext());
@@ -36,7 +38,8 @@ public class T9Preferences {
 		return self;
 	}
 
-	/************* VALIDATORS *************/
+
+	/************* validators *************/
 
 	private boolean doesLanguageExist(int langId) {
 		return LanguageCollection.getLanguage(langId) != null;
@@ -70,7 +73,7 @@ public class T9Preferences {
 	}
 
 
-	/************* PREFERENCES OPERATIONS *************/
+	/************* input settings *************/
 
 	public ArrayList<Integer> getEnabledLanguages() {
 		int languageMask = prefs.getInt("pref_enabled_languages", 1);
@@ -101,6 +104,7 @@ public class T9Preferences {
 		prefsEditor.apply();
 	}
 
+
 	public int getTextCase() {
 		return prefs.getInt("pref_text_case", InputMode.CASE_LOWER);
 	}
@@ -131,6 +135,7 @@ public class T9Preferences {
 		}
 	}
 
+
 	public int getInputMode() {
 		return prefs.getInt("pref_input_mode", InputMode.MODE_PREDICTIVE);
 	}
@@ -146,18 +151,24 @@ public class T9Preferences {
 	}
 
 
+	/************* hotkey settings *************/
+
 	public int getKeyBackspace() {
 		return prefs.getInt("pref_key_backspace", KeyEvent.KEYCODE_BACK);
 	}
-
 	public int getKeyInputMode() { return prefs.getInt("pref_key_input_mode", KeyEvent.KEYCODE_POUND); }
-
 	public int getKeyOtherActions() { return prefs.getInt("pref_key_other_actions", KeyEvent.KEYCODE_STAR); }
 
 
-	public int getSuggestionsMin() { return 8; }
-	public int getSuggestionsMax() { return 20; }
+	/************* internal settings *************/
 
+	public int getDictionaryImportProgressUpdateInterval() { return 100; /* ms */ }
+	public int getDictionaryImportWordChunkSize() { return 1000; /* words */ }
+	public int getSuggestionsMax() { return 20; }
+	public int getSuggestionsMin() { return 8; }
+
+
+	/************* add word, last word *************/
 
 	public String getLastWord() {
 		return prefs.getString("last_word", "");
@@ -173,5 +184,4 @@ public class T9Preferences {
 	public void clearLastWord() {
 		this.saveLastWord("");
 	}
-
 }

src/io/github/sspanak/tt9/ui/TraditionalT9Settings.java

@@ -1,6 +1,5 @@
 package io.github.sspanak.tt9.ui;
 
-import android.app.Activity;
 import android.app.AlertDialog;
 import android.app.ListActivity;
 import android.app.ProgressDialog;
@@ -8,38 +7,24 @@ import android.content.Context;
 import android.content.DialogInterface;
 import android.content.Intent;
 import android.net.Uri;
-import android.os.AsyncTask;
 import android.os.Bundle;
-import android.os.Environment;
 import android.os.Handler;
 import android.os.Looper;
 import android.os.Message;
-import android.os.SystemClock;
-import android.text.TextUtils;
 import android.view.View;
 import android.widget.ListAdapter;
 import android.widget.ListView;
 
-import com.stackoverflow.answer.UnicodeBOMInputStream;
-
-import java.io.BufferedReader;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Properties;
 
-import io.github.sspanak.tt9.Logger;
 import io.github.sspanak.tt9.R;
 import io.github.sspanak.tt9.db.DictionaryDb;
-import io.github.sspanak.tt9.db.Word;
+import io.github.sspanak.tt9.db.DictionaryImportException;
+import io.github.sspanak.tt9.db.DictionaryLoader;
+import io.github.sspanak.tt9.languages.InvalidLanguageCharactersException;
+import io.github.sspanak.tt9.languages.InvalidLanguageException;
 import io.github.sspanak.tt9.languages.Language;
 import io.github.sspanak.tt9.languages.LanguageCollection;
 import io.github.sspanak.tt9.preferences.T9Preferences;
@@ -49,365 +34,11 @@ import io.github.sspanak.tt9.settings_legacy.SettingAdapter;
 
 public class TraditionalT9Settings extends ListActivity implements DialogInterface.OnCancelListener {
 
-	AsyncTask<String, Integer, Reply> task = null;
-	final static String userdictname = "user.%s.dict";
-	final static String sddir = "tt9";
+	private DictionaryLoader loader;
+	ProgressDialog progressDialog;
 
 	Context mContext = null;
 
-	public static class LoadException extends Exception {
-		private static final long serialVersionUID = 3323913652550046354L;
-
-		public LoadException() {
-			super();
-		}
-	}
-
-	private static class Reply {
-		public boolean status;
-		private final List<String> msgs;
-
-		protected Reply() {
-			this.status = true;
-			this.msgs = new ArrayList<>(4);
-		}
-
-		protected void addMsg(String msg) throws LoadException {
-			msgs.add(msg);
-			if (msgs.size() > 6) {
-				msgs.add("Too many errors, bailing.");
-				throw new LoadException();
-			}
-		}
-		protected void forceMsg(String msg) {
-			msgs.add(msg);
-		}
-
-	}
-
-	private void finishAndShowError(ProgressDialog pd, Reply result, int title){
-		if (pd != null) {
-			// Logger.d("onPostExecute", "pd");
-			if (pd.isShowing()) {
-				pd.dismiss();
-			}
-		}
-		if (result == null) {
-			// bad thing happened
-			Logger.e("onPostExecute", "Bad things happen?");
-		} else {
-			String msg = TextUtils.join("\n", result.msgs);
-			Logger.d("onPostExecute", "Result: " + result.status + " " + msg);
-			if (!result.status) {
-				showErrorDialog(getResources().getString(title), msg);
-			}
-		}
-	}
-
-	private static void closeStream(Closeable is, Reply reply) {
-		if (is == null) {
-			return;
-		}
-		try {
-			is.close();
-		} catch (IOException e) {
-			e.printStackTrace();
-			reply.forceMsg("Couldn't close stream: " + e.getMessage());
-		}
-	}
-
-	private class LoadDictTask extends AsyncTask<String, Integer, Reply> {
-		/**
-		 * The system calls this to perform work in a worker thread and delivers
-		 * it the parameters given to AsyncTask.execute()
-		 */
-		ProgressDialog pd;
-		long size;
-		long pos;
-		boolean internal;
-		String[] dicts;
-		ArrayList<Language> mSupportedLanguages;
-
-		LoadDictTask(int msgid, boolean intern, ArrayList<Language> supportedLanguages) {
-			internal = intern;
-
-			dicts = new String[supportedLanguages.size()];
-			int x = 0;
-			for (Language language : supportedLanguages) {
-				if (intern) {
-					dicts[x++] = language.getDictionaryFile();
-				} else {
-					dicts[x++] = String.format(userdictname, language.getName().toLowerCase(Locale.ENGLISH));
-				}
-			}
-			mSupportedLanguages = supportedLanguages;
-
-			pd = new ProgressDialog(TraditionalT9Settings.this);
-			pd.setMessage(getResources().getString(msgid));
-			pd.setOnCancelListener(TraditionalT9Settings.this);
-		}
-
-		private long getDictSizes(boolean internal, String[] dicts) {
-			if (internal) {
-				InputStream input;
-				Properties props = new Properties();
-				try {
-					input = getAssets().open("dict.properties");
-					props.load(input);
-					long total = 0;
-					for (String dict : dicts) {
-						total += Long.parseLong(props.getProperty("size." + dict));
-					}
-					return total;
-
-				} catch (IOException e) {
-					Logger.e("getDictSizes", "Unable to get dict sizes");
-					e.printStackTrace();
-					return -1;
-				} catch (NumberFormatException e) {
-					Logger.e("getDictSizes", "Unable to parse sizes");
-					return -1;
-				}
-			} else {
-				File backupfile = new File(Environment.getExternalStorageDirectory(), sddir);
-				long total = 0;
-				File f;
-				for (String dict : dicts) {
-					f = new File(backupfile, dict);
-					if (f.exists() && f.isFile()) {
-						total = total + f.length();
-					} else {
-						total = total + 0;
-					}
-				}
-				return total;
-			}
-		}
-
-		@Override protected void onPreExecute() {
-			size = getDictSizes(internal, dicts);
-			pos = 0;
-			if ( size >= 0 ) {
-				pd.setProgressStyle(ProgressDialog.STYLE_HORIZONTAL);
-				pd.setMax(10000);
-			} else {
-				pd.setProgressStyle(ProgressDialog.STYLE_SPINNER);
-			}
-			pd.show();
-		}
-
-		@Override
-		protected Reply doInBackground(String... mode) {
-			Reply reply = new Reply();
-
-			long startnow, endnow;
-			startnow = SystemClock.uptimeMillis();
-
-			// add characters first, then dictionary:
-			Logger.d("doInBackground", "Adding characters...");
-			processChars(mSupportedLanguages);
-			Logger.d("doInBackground", "Characters added.");
-
-			Logger.d("doInBackground", "Adding dict(s)...");
-
-			InputStream dictstream = null;
-
-			try {
-				for (int x=0; x<dicts.length; x++) {
-					if (internal) {
-						try {
-							dictstream = getAssets().open(dicts[x]);
-							reply = processFile(dictstream, reply, mSupportedLanguages.get(x), dicts[x]);
-						} catch (IOException e) {
-							e.printStackTrace();
-							reply.status = false;
-							reply.forceMsg("IO Error: " + e.getMessage());
-						}
-					} else {
-						try {
-							dictstream = new FileInputStream(new File(
-									new File(Environment.getExternalStorageDirectory(), sddir),	dicts[x]));
-							reply = processFile(dictstream, reply, mSupportedLanguages.get(x), dicts[x]);
-						} catch (FileNotFoundException e) {
-							reply.status = false;
-							reply.forceMsg("File not found: " + e.getMessage());
-							final String msg = mContext.getString(R.string.dictionary_not_found, dicts[x]);
-							//Logger.d("T9Setting.load", "Built string. Calling Toast.");
-							((Activity) mContext).runOnUiThread(new Runnable() {
-								@Override
-								public void run() {
-									UI.toast(mContext, msg);
-								}
-							});
-
-							closeStream(dictstream, reply); // this is silly but it
-							// stops IDE nagging at me.
-						} catch (IOException e) {
-							reply.status = false;
-							reply.forceMsg("IO Error: " + e.getMessage());
-							closeStream(dictstream, reply); // this is silly but it
-							return reply;					// stops IDE nagging at me.
-						}
-					}
-					closeStream(dictstream, reply);
-				}
-			} catch (LoadException e) {
-				// too many errors, bail
-				closeStream(dictstream, reply);
-			}
-			endnow = SystemClock.uptimeMillis();
-			Logger.d("TIMING", "Execution time: " + (endnow - startnow) + " ms");
-			return reply;
-		}
-
-		/**
-		 * processChars
-		 * Inserts single characters.
-		 */
-		private void processChars(List<Language> allLanguages) {
-			ArrayList<Word> list = new ArrayList<>();
-
-			try {
-				for (Language lang : allLanguages) {
-					for (int key = 0; key <= 9; key++) {
-						for (String langChar : lang.getKeyCharacters(key)) {
-							if (langChar.length() == 1 && langChar.charAt(0) >= '0' && langChar.charAt(0) <= '9') {
-								// We do not want 0-9 as "word suggestions" in Predictive mode. It looks confusing
-								// when trying to type a word and also, one can type them by holding the respective
-								// key.
-								continue;
-							}
-
-							Word word = new Word();
-							word.langId = lang.getId();
-							word.sequence = String.valueOf(key);
-							word.word = langChar;
-							word.frequency = 0;
-
-							list.add(word);
-						}
-					}
-				}
-
-				DictionaryDb.insertWordsSync(list);
-			} catch (Exception e) {
-				Logger.e("processChars", e.getMessage());
-			}
-		}
-
-		private String getLine(BufferedReader br, Reply rpl, String fname) throws LoadException {
-			try {
-				return br.readLine();
-			} catch (IOException e) {
-				e.printStackTrace();
-				rpl.status = false;
-				rpl.addMsg("IO Error ("+fname+"): " + e.getMessage());
-			}
-			return null;
-		}
-
-		private Reply processFile(InputStream is, Reply rpl, Language lang, String fname)
-				throws LoadException, IOException {
-			long start = System.currentTimeMillis();
-
-			UnicodeBOMInputStream ubis = new UnicodeBOMInputStream(is);
-
-			BufferedReader br = new BufferedReader(new InputStreamReader(ubis));
-			ubis.skipBOM();
-
-			int freq;
-			String seq;
-			int linecount = 1;
-			int wordlen;
-			String fileWord = getLine(br, rpl, fname);
-			ArrayList<Word> dbWords = new ArrayList<>();
-			int insertChunkSize = 1000;
-			int progressUpdateInterval = 100; // ms
-			long lastProgressUpdate = 0;
-
-			try {
-
-				DictionaryDb.beginTransaction();
-
-				while (fileWord != null) {
-					if (isCancelled()) {
-						rpl.status = false;
-						rpl.addMsg("User cancelled.");
-						break;
-					}
-					if (fileWord.contains(" ")) {
-						rpl.status = false;
-						rpl.addMsg("Cannot parse word with spaces: " + fileWord);
-						break;
-					}
-
-					freq = 0;
-					wordlen = fileWord.getBytes(StandardCharsets.UTF_8).length;
-					pos += wordlen;
-					// replace junk characters:
-					fileWord = fileWord.replace("\uFEFF", "");
-					try {
-						seq = lang.getDigitSequenceForWord(fileWord);
-					} catch (Exception e) {
-						rpl.status = false;
-						rpl.addMsg("Error on word ("+fileWord+") line "+
-								linecount+" in (" +	fname+"): "+
-								getResources().getString(R.string.add_word_badchar, lang.getName(), fileWord));
-						break;
-					}
-					linecount++;
-
-					Word word = new Word();
-					word.sequence = seq;
-					word.langId = lang.getId();
-					word.word = fileWord;
-					word.frequency = freq;
-					dbWords.add(word);
-
-					if (linecount % insertChunkSize == 0) {
-						DictionaryDb.insertWordsSync(dbWords);
-						dbWords.clear();
-					}
-
-					if (size >= 0 && System.currentTimeMillis() - lastProgressUpdate > progressUpdateInterval) {
-						publishProgress((int) ((float) pos / size * 10000));
-						lastProgressUpdate = System.currentTimeMillis();
-					}
-					fileWord = getLine(br, rpl, fname);
-				}
-
-				DictionaryDb.insertWordsSync(dbWords);
-				DictionaryDb.endTransaction(true);
-				dbWords.clear();
-
-				publishProgress((int) ((float) pos / size * 10000));
-			} catch (Exception e) {
-				DictionaryDb.endTransaction(false);
-				Logger.e("processFile", e.getMessage());
-			}	finally {
-				br.close();
-				is.close();
-				ubis.close();
-				is.close();
-
-				Logger.d("processFile", "Inserted: " + fname + " in: " + (System.currentTimeMillis() - start) + "ms");
-			}
-			return rpl;
-		}
-
-		@Override
-		protected void onProgressUpdate(Integer... progress) {
-			if (pd.isShowing()) {
-				pd.setProgress(progress[0]);
-			}
-		}
-
-		@Override
-		protected void onPostExecute(Reply result) {
-			finishAndShowError(pd, result, R.string.dictionary_load_title);
-		}
-	}
-
 	@Override
 	protected void onCreate(Bundle savedInstanceState) {
 		super.onCreate(savedInstanceState);
@@ -433,22 +64,33 @@ public class TraditionalT9Settings extends ListActivity implements DialogInterfa
 
 	}
 
+	@Override
+	public void onCancel(DialogInterface dint) {
+		if (loader != null) {
+			loader.stop();
+		}
+	}
+
 	@Override
 	protected  void onListItemClick(ListView l, View v, int position, long id) {
 		Setting s = (Setting)getListView().getItemAtPosition(position);
-		if (s.id.equals("help"))
-			openHelp();
-		else if (s.id.equals("loaddict"))
-			preloader(R.string.dictionary_loading, true);
-		else if (s.id.equals("truncatedict")) {
-			truncateWords();
+		switch (s.id) {
+			case "help":
+				openHelp();
+				break;
+			case "loaddict":
+				loadDictionaries();
+				break;
+			case "truncatedict":
+				truncateWords();
+				break;
+			default:
+				s.clicked(mContext);
+				break;
 		}
-		else if (s.id.equals("loaduserdict"))
-			preloader(R.string.dictionary_loading_user_dict, false);
-		else
-			s.clicked(mContext);
 	}
 
+
 	private void openHelp() {
 		Intent i = new Intent(Intent.ACTION_VIEW);
 		i.setData(Uri.parse(getString(R.string.help_url)));
@@ -465,50 +107,106 @@ public class TraditionalT9Settings extends ListActivity implements DialogInterfa
 			DictionaryDb.truncateWords(afterTruncate);
 	}
 
+	private void loadDictionaries() {
+		ArrayList<Language> languages = LanguageCollection.getAll(T9Preferences.getInstance().getEnabledLanguages());
+		initProgress(100 * languages.size());
 
-	private void preloader(int msgid, boolean internal) {
+		Handler loadHandler = new Handler(Looper.getMainLooper()) {
+			@Override
+			public void handleMessage(Message msg) {
+				String error = msg.getData().getString("error", null);
 
+				if (error != null) {
+					hideProgress();
+					handleError(
+						error,
+						msg.getData().getInt("languageId", -1),
+						msg.getData().getLong("fileLine", -1),
+						msg.getData().getString("word", "")
+					);
+				} else {
+					int langId = msg.getData().getInt("languageId", -1);
+					Language lang = LanguageCollection.getLanguage(langId);
+					String langName = lang != null ? lang.getName() : "???";
 
-		task = new LoadDictTask(
-			msgid,
-			internal,
-			LanguageCollection.getAll(T9Preferences.getInstance().getEnabledLanguages())
-		);
-		task.execute();
+					String title = getResources().getString(R.string.dictionary_loading, langName);
+					showProgress(
+						msg.getData().getInt("currentFile", 0),
+						msg.getData().getInt("progress", 0),
+						title
+					);
+				}
+			}
+		};
+
+		loader = new DictionaryLoader(this);
+		loader.load(loadHandler, languages);
 	}
 
 
+	private void initProgress(int max) {
+		if (progressDialog == null) {
+			progressDialog = new ProgressDialog(this);
+			progressDialog.setOnCancelListener(TraditionalT9Settings.this);
+			progressDialog.setProgressStyle(ProgressDialog.STYLE_HORIZONTAL);
+		}
+
+		progressDialog.setMax(max);
+	}
+
+	private void showProgress(int currentFile, int currentFileProgress, String title) {
+		if (progressDialog == null) {
+			return;
+		}
+
+		if (title != null) {
+			progressDialog.setMessage(title);
+		}
+
+		int totalProgress = 100 * currentFile + currentFileProgress;
+		if (totalProgress <= 0 || totalProgress >= progressDialog.getMax()) {
+			progressDialog.dismiss();
+		} else {
+			progressDialog.setProgress(totalProgress);
+			if (!progressDialog.isShowing()) {
+				progressDialog.show();
+			}
+		}
+	}
+
+	private void hideProgress() {
+		if (progressDialog != null) {
+			progressDialog.dismiss();
+		}
+	}
+
+	private void handleError(String errorType, int langId, long line, String word) {
+		Language lang = LanguageCollection.getLanguage(langId);
+		String message;
+
+		if (lang == null || errorType.equals(InvalidLanguageException.class.getSimpleName())) {
+			message = getString(R.string.add_word_invalid_language);
+		} else if (errorType.equals(DictionaryImportException.class.getSimpleName()) || errorType.equals(InvalidLanguageCharactersException.class.getSimpleName())) {
+			String languageName = lang.getName();
+			message = getString(R.string.dictionary_import_bad_char, word, line, languageName);
+		} else if (errorType.equals(IOException.class.getSimpleName()) || errorType.equals(FileNotFoundException.class.getSimpleName())) {
+			String languageName = lang.getName();
+			message = getString(R.string.dictionary_not_found, languageName);
+		} else {
+			String languageName = lang.getName();
+			message = getString(R.string.dictionary_import_error, languageName, errorType);
+		}
+
+		showErrorDialog(getString(R.string.dictionary_load_title), message);
+	}
+
 	private void showErrorDialog(CharSequence title, CharSequence msg) {
-		showErrorDialog(new AlertDialog.Builder(this), title, msg);
-	}
-
-	private void showErrorDialog(AlertDialog.Builder builder, CharSequence title, CharSequence msg) {
-		builder.setMessage(msg).setTitle(title)
-				.setNeutralButton(android.R.string.ok, new DialogInterface.OnClickListener() {
-					@Override
-					public void onClick(DialogInterface dialog, int id) {
-						dialog.dismiss();
-					}
-				});
+		AlertDialog.Builder builder = new AlertDialog.Builder(this);
+		builder
+			.setMessage(msg)
+			.setTitle(title)
+			.setNeutralButton(android.R.string.ok, (dialog, id) -> dialog.dismiss());
 		AlertDialog dialog = builder.create();
 		dialog.show();
 	}
-
-	private void showErrorDialogID(AlertDialog.Builder builder, int titleid, int msgid) {
-		builder.setMessage(msgid).setTitle(titleid)
-				.setNeutralButton(android.R.string.ok, new DialogInterface.OnClickListener() {
-					@Override
-					public void onClick(DialogInterface dialog, int id) {
-						dialog.dismiss();
-					}
-				});
-		AlertDialog dialog = builder.create();
-		dialog.show();
-	}
-
-
-	@Override
-	public void onCancel(DialogInterface dint) {
-		task.cancel(false);
-	}
 }