diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c1ffdce..88bb1723 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,10 +24,12 @@ jobs: uses: gradle/gradle-build-action@v3 # validate and build + - name: Validate Help + run: ./gradlew convertHelp - name: Validate Dictionaries run: ./gradlew validateLanguages - name: Build Languages - run: ./gradlew copyDefinitions copyDictionaries writeDictionaryProperties + run: ./gradlew copyDefinitions convertHelp buildDictionaryDownloads copyDownloadsToAssets - name: Lint run: ./gradlew lint # this actually runs mergeResources, so it must come after the dictionary tasks - name: Build all APK variants diff --git a/app/build-dictionary.gradle b/app/build-dictionary.gradle new file mode 100644 index 00000000..a596c5ff --- /dev/null +++ b/app/build-dictionary.gradle @@ -0,0 +1,203 @@ +import java.nio.charset.StandardCharsets +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +apply from: 'dictionary-tools.gradle' + +ext.convertDictionaries = { definitionsInputDir, dictionariesInputDir, dictionariesOutputDir, dictionariesMetaDir -> + int errorCount = 0 + + def errorStream = fileTree(dir: definitionsInputDir).getFiles().parallelStream().map { definition -> + def (_, sounds, __, locale, dictionaryFile, langFileErrorCount, langFileErrorMsg) = parseLanguageDefintion(definition, dictionariesInputDir) + errorCount += langFileErrorCount + if (!langFileErrorMsg.isEmpty()) { + return langFileErrorMsg + } + + def (conversionErrorCount, conversionErrorMessages) = convertDictionary(definition, dictionaryFile, dictionariesOutputDir, dictionariesMetaDir, DICTIONARY_OUTPUT_EXTENSION, sounds, locale, MAX_ERRORS, CSV_DELIMITER) + errorCount += conversionErrorCount + if (!conversionErrorMessages.isEmpty()) { + return conversionErrorMessages + } + + return "" + } + + String errorsMsg = errorStream.reduce("", String::concat) + if (errorsMsg) { + throw new GradleException(errorsMsg) + } +} + + +// this cannot be static, because DictionaryTools will not be visible +def convertDictionary(File definition, File csvDictionary, String dictionariesOutputDir, String dictionariesMetaDir, String outputDictionaryExtension, HashMap sounds, Locale locale, int maxErrors, String csvDelimiter) { + if (isDictionaryUpToDate(definition, csvDictionary, dictionariesMetaDir)) { + return [0, ""] + } + + + int errorCount = 0 + String errorMsg = '' + + List fileContents = csvDictionary.readLines() + LinkedHashMap> outputDictionary = new LinkedHashMap<>() + int wordCount = 0 + + for (int lineNumber = 1; lineNumber <= fileContents.size() && errorCount < maxErrors; lineNumber++) { + String line = fileContents.get(lineNumber - 1) + + def (word, transcription, frequency) = DictionaryTools.getDictionaryLineData(line, csvDelimiter) + + String digitSequence = "" + try { + def transcribedWord = transcription.isEmpty() ? word : transcription + digitSequence = DictionaryTools.wordToDigitSequence(locale, transcribedWord, sounds, !transcription.isEmpty()) + } catch (IllegalArgumentException e) { + errorCount++ + errorMsg += "Dictionary '${csvDictionary.name}' is invalid. Failed generating digit sequence for word '${word}' on line ${lineNumber}. ${e.message}\n" + } + + if (errorCount == 0) { + if (!outputDictionary.containsKey(digitSequence)) { + outputDictionary.put(digitSequence, new ArrayList<>()) + } + // prefix the frequency to sort the words later + outputDictionary.get(digitSequence).add("${String.format('%03d', frequency)}${word}") + wordCount++ + } + } + + outputDictionary = sortDictionary(outputDictionary) + + def (assetError, zippedDictionary) = writeZippedDictionary(dictionariesOutputDir, csvDictionary, outputDictionary, outputDictionaryExtension) + if (assetError) { + errorCount++ + errorMsg += assetError + } + + def propertiesError = writeDictionaryProperties(definition, csvDictionary, zippedDictionary, dictionariesMetaDir, outputDictionary.size(), wordCount) + if (propertiesError) { + errorCount++ + errorMsg += propertiesError + } + + return [errorCount, errorMsg] +} + + +//////////////////// DICTIONARY PROCESSING //////////////////// + +static byte[] compressDictionaryLine(String digitSequence, List words) { + if (words.isEmpty()) { + throw new IllegalArgumentException("No words for digit sequence: ${digitSequence}") + } + + boolean shouldSeparateWords = false + + for (def i = 0; i < words.size(); i++) { + if (words.get(i).length() != digitSequence.length()) { + shouldSeparateWords = true + break + } + } + + return ( + digitSequence + + (shouldSeparateWords ? ' ' : '') + + words.join(shouldSeparateWords ? ' ' : null) + ).getBytes(StandardCharsets.UTF_8) +} + + +def isDictionaryUpToDate(File definition, File csvDictionary, String dictionaryPropertiesDir) { + def dictionaryProperties = new File(dictionaryPropertiesDir, getPropertyFileName(csvDictionary)) + if (!dictionaryProperties.exists()) { + return false + } + + Properties props = new Properties() + dictionaryProperties.withInputStream { stream -> props.load(stream) } + + return props.getProperty("hash", "") == DictionaryTools.getLanguageHash(definition, csvDictionary) +} + + +/** + * Sorts the dictionary in ascending order of sequence length and in descending order of word frequency. + * Also, it removes the frequency prefix from each word. The input dictionary is not modified. + */ +static LinkedHashMap> sortDictionary(LinkedHashMap> dictionary) { + // sort the sequences in ascending order of length, then lexicographically + def sequences = dictionary.keySet().toList() + Collections.sort(sequences, { a, b -> + a.length() == b.length() ? a.compareTo(b) : a.length() - b.length() + }) + def sortedDictionary = new LinkedHashMap>() + sequences.each { sequence -> sortedDictionary.put(sequence, dictionary.get(sequence)) } + + // sort the words for each sequence in descending order of frequency + sortedDictionary.forEach { _, words -> { + Collections.sort(words, Collections.reverseOrder()) + words.replaceAll { word -> word.replaceFirst("^\\d+", "") } + }} + + return sortedDictionary +} + + +//////////////////// FILE I/O //////////////////// + +static getDictionaryFileName(csvDictionary) { + return "${csvDictionary.getName().replaceFirst("\\.\\w+\$", "")}" +} + + +static getPropertyFileName(csvDictionary) { + return "${getDictionaryFileName(csvDictionary)}.props.yml" +} + + +static getZipDictionaryFile(dictionariesOutputDir, csvDictionary, outputDictionaryExtension) { + return new File(dictionariesOutputDir, "${getDictionaryFileName(csvDictionary)}.${outputDictionaryExtension}") +} + + +/** + * Zipping the text files results in a smaller APK in comparison to the uncompressed text files. + */ +static def writeZippedDictionary(dictionariesOutputDir, csvDictionaryFile, outputDictionary, outputDictionaryExtension) { + def fileName = getDictionaryFileName(csvDictionaryFile) + def outputFile = getZipDictionaryFile(dictionariesOutputDir, csvDictionaryFile, outputDictionaryExtension) + + try { + def zipOutputStream = new ZipOutputStream(new FileOutputStream(outputFile)) + zipOutputStream.putNextEntry(new ZipEntry("${fileName}.txt")) + outputDictionary.each { digitSequence, words -> + zipOutputStream.write(compressDictionaryLine(digitSequence, words)) + } + zipOutputStream.closeEntry() + zipOutputStream.close() + return ["", outputFile] + } catch (Exception e) { + return ["Failed writing to '${outputFile.path}'. ${e.message}\n", outputFile] + } +} + + +// this cannot be static, because it requires access to exec() and DictionaryTools +def writeDictionaryProperties(File definition, File csvDictionary, File zipDictionary, outputDir, int sequences, int words) { + def name = getPropertyFileName(csvDictionary) + + try { + def hash = DictionaryTools.getLanguageHash(definition, csvDictionary) + def revision = zipDictionary.exists() ? exec("git log --pretty=tformat:%H -n 1 ${zipDictionary}") : "" + def size = zipDictionary.exists() ? zipDictionary.length() : 0 + + new File(outputDir, name).text = "hash: ${hash}\nrevision: ${revision}\nsequences: ${sequences}\nsize: ${size}\nwords: ${words}" + + return "" + } catch (Exception e) { + return "Failed writing dictionary properties to: '${outputDir}/${name}'. ${e.message}\n" + } +} diff --git a/app/build.gradle b/app/build.gradle index 7eafc4e9..2afe501d 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -3,12 +3,19 @@ plugins { } apply from: 'constants.gradle' -apply from: 'dictionary-tools.gradle' -apply from: 'help-tools.gradle' +apply from: 'build-dictionary.gradle' apply from: 'validate-languages.gradle' +apply from: 'help-tools.gradle' apply from: 'version-tools.gradle' +tasks.register('copyDefinitions', Copy) { + from LANGUAGES_INPUT_DIR + include '**/*.yml' + into LANGUAGES_OUTPUT_DIR +} + + tasks.register('validateLanguages') { inputs.dir LANGUAGES_INPUT_DIR outputs.dir LANGUAGE_VALIDATION_DIR @@ -18,19 +25,30 @@ tasks.register('validateLanguages') { } } -tasks.register('copyDefinitions', Copy) { - from LANGUAGES_INPUT_DIR - include '**/*.yml' - into LANGUAGES_OUTPUT_DIR + +tasks.register('buildDictionaryDownloads') { + inputs.dir DICTIONARIES_INPUT_DIR + outputs.dir DICTIONARIES_DOWNLOAD_DIR + outputs.dir DICTIONARY_META_OUTPUT_DIR + + dependsOn validateLanguages + mustRunAfter validateLanguages + + doLast { + convertDictionaries(DEFINITIONS_INPUT_DIR, DICTIONARIES_INPUT_DIR, DICTIONARIES_DOWNLOAD_DIR, DICTIONARY_META_OUTPUT_DIR) + } } -tasks.register('copyDictionaries', Copy) { - from DICTIONARIES_INPUT_DIR - include '**/*.csv' - include '**/*.txt' + +tasks.register('copyDownloadsToAssets', Copy) { + from DICTIONARIES_DOWNLOAD_DIR + include '**/*.zip' into DICTIONARIES_OUTPUT_DIR + dependsOn buildDictionaryDownloads + mustRunAfter buildDictionaryDownloads } + tasks.register('convertHelp') { inputs.dir HELP_MARKDOWN_DIR outputs.dir HELP_HTML_DIR @@ -40,15 +58,6 @@ tasks.register('convertHelp') { } } -tasks.register('writeDictionaryProperties') { - inputs.dir fileTree(dir: DICTIONARIES_INPUT_DIR) - outputs.dir DICTIONARY_META_OUTPUT_DIR - - doLast { - getDictionaryProperties(DICTIONARIES_INPUT_DIR, DICTIONARY_META_OUTPUT_DIR) - } -} - tasks.register('updateManifest') { doLast { updateManifestVersion(getVersionCode(), getVersionName()) @@ -58,6 +67,7 @@ tasks.register('updateManifest') { clean { delete LANGUAGES_OUTPUT_DIR delete DICTIONARIES_OUTPUT_DIR + delete DICTIONARIES_DOWNLOAD_DIR delete HELP_HTML_DIR } @@ -84,10 +94,12 @@ android { } buildTypes { debug { + buildConfigField 'String', 'DICTIONARY_EXTENSION', "\"${DICTIONARY_OUTPUT_EXTENSION}\"" buildConfigField 'String', 'VERSION_FULL', "\"${getVersionString('debug')}\"" } release { + buildConfigField 'String', 'DICTIONARY_EXTENSION', "\"${DICTIONARY_OUTPUT_EXTENSION}\"" buildConfigField 'String', 'VERSION_FULL', "\"${getVersionString('release')}\"" debuggable false @@ -124,7 +136,11 @@ android { ].each { taskName -> try { tasks.named(taskName)?.configure { - dependsOn(validateLanguages, copyDefinitions, copyDictionaries, writeDictionaryProperties, convertHelp) + dependsOn(copyDefinitions, convertHelp, validateLanguages, buildDictionaryDownloads) + } + + if (taskName.toLowerCase().contains("full")) { + tasks.named(taskName)?.configure {dependsOn(copyDownloadsToAssets) } } } catch (UnknownTaskException ignored) {} } diff --git a/app/constants.gradle b/app/constants.gradle index 4c96c664..aca5916d 100644 --- a/app/constants.gradle +++ b/app/constants.gradle @@ -4,19 +4,22 @@ ext.PACKAGE_NAME = "io.github.sspanak.${APP_NAME}" ext.LANGUAGES_DIR_NAME = 'languages' ext.DEFINITIONS_DIR_NAME = 'definitions' ext.DICTIONARIES_DIR_NAME = 'dictionaries' +ext.DICTIONARIES_DOWNLOAD_DIR_NAME = 'downloads' ext.DICTIONARY_SIZES_DIR_NAME = 'dictionary-sizes' -def ROOT_DIR = "${project.rootDir}/app" -def MAIN_ASSETS_DIR = "${ROOT_DIR}/src/main/assets" -def FULL_VERSION_ASSETS_DIR = "${ROOT_DIR}/src/full/assets" +def APP_ROOT_DIR = "${project.rootDir}/app" +def MAIN_ASSETS_DIR = "${APP_ROOT_DIR}/src/main/assets" +def FULL_VERSION_ASSETS_DIR = "${APP_ROOT_DIR}/src/full/assets" ext.HELP_MARKDOWN_DIR = "${project.rootDir}/docs/help" ext.HELP_HTML_DIR = "${MAIN_ASSETS_DIR}/help" -ext.LANGUAGES_INPUT_DIR = "${ROOT_DIR}/${LANGUAGES_DIR_NAME}" +ext.LANGUAGES_INPUT_DIR = "${APP_ROOT_DIR}/${LANGUAGES_DIR_NAME}" ext.DEFINITIONS_INPUT_DIR = "${LANGUAGES_INPUT_DIR}/${DEFINITIONS_DIR_NAME}" ext.DICTIONARIES_INPUT_DIR = "${LANGUAGES_INPUT_DIR}/${DICTIONARIES_DIR_NAME}" +ext.DICTIONARIES_DOWNLOAD_DIR = "${project.rootDir}/${DICTIONARIES_DOWNLOAD_DIR_NAME}" + ext.LANGUAGES_OUTPUT_DIR = "${MAIN_ASSETS_DIR}/${LANGUAGES_DIR_NAME}" ext.DEFINITIONS_OUTPUT_DIR = "${LANGUAGES_OUTPUT_DIR}/${DEFINITIONS_DIR_NAME}" ext.DICTIONARY_META_OUTPUT_DIR = "${LANGUAGES_OUTPUT_DIR}/${DICTIONARIES_DIR_NAME}" @@ -25,5 +28,6 @@ ext.DICTIONARIES_OUTPUT_DIR = "${FULL_VERSION_ASSETS_DIR}/${LANGUAGES_DIR_NAME}/ ext.LANGUAGE_VALIDATION_DIR = layout.buildDirectory.dir("langValidation") ext.CSV_DELIMITER = ' ' // TAB +ext.DICTIONARY_OUTPUT_EXTENSION = 'zip' ext.MAX_WORD_FREQUENCY = 255 ext.MAX_ERRORS = 50 diff --git a/app/dictionary-tools.gradle b/app/dictionary-tools.gradle index 2e139cbd..c9c1b80f 100644 --- a/app/dictionary-tools.gradle +++ b/app/dictionary-tools.gradle @@ -1,10 +1,65 @@ -ext.getDictionaryProperties = { dictionariesDir, sizesDir -> - fileTree(dir: dictionariesDir).getFiles().parallelStream().forEach {dictionary -> - def hash = dictionary.exists() ? dictionary.text.digest("SHA-1") : "" - def revision = dictionary.exists() ? exec("git log --pretty=tformat:%H -n 1 ${dictionary}") : "" - def size = dictionary.exists() ? dictionary.length() : 0 - def words = dictionary.exists() ? dictionary.text.split("\n").length : 0 +class Wrapper { + static def getDictionaryLineData(String line, String delimiter) { + String[] parts = line.split(delimiter, 2) + String word = parts[0] + String transcription = parts.length > 1 && parts[1] =~ "^[a-zA-Z]+\$" ? parts[1] : "" - new File(sizesDir, "${dictionary.getName()}.props.yml").text = "hash: ${hash}\nrevision: ${revision}\nsize: ${size}\nwords: ${words}" - } -} \ No newline at end of file + int frequency + try { + int partsElement = transcription.isEmpty() ? 1 : 2 + frequency = (parts.length > partsElement ? parts[partsElement] : "0") as int + } catch (Exception ignored) { + frequency = -1 + } + + return [word, transcription, frequency] + } + + + static def wordToDigitSequence(Locale locale, String word, HashMap sounds, boolean isTranscribed) { + String sequence = "" + + final String normalizedWord = isTranscribed ? word : word.toUpperCase(locale) + String currentSound = "" + + for (int i = 0, end = normalizedWord.length() - 1; i <= end; i++) { + char currentChar = normalizedWord.charAt(i) + char nextChar = i < end ? normalizedWord.charAt(i + 1) : 0 + int nextCharType = Character.getType(nextChar) + + currentSound += currentChar + + // charAt(i) returns "ΐ" as three separate characters, but they must be treated as one. + if ( + locale.getLanguage() == "el" + && (nextCharType == Character.NON_SPACING_MARK || nextCharType == Character.ENCLOSING_MARK || nextCharType == Character.COMBINING_SPACING_MARK) + ) { + continue + } + + if (!isTranscribed || i == end || Character.isUpperCase(nextChar)) { + if (!sounds.containsKey(currentSound)) { + throw new IllegalArgumentException("Sound or layout entry '${currentSound}' does not belong to the language sound list: ${sounds}.") + } else { + sequence += sounds.get(currentSound) + currentSound = "" + } + } + } + + if (sequence.isEmpty()) { + throw new IllegalArgumentException("The word does not contain any valid sounds.") + } + + return sequence + } + + + static def getLanguageHash(File definitionFile, File dictionaryFile) { + def definitionHash = definitionFile != null && definitionFile.exists() ? definitionFile.text.digest("SHA-256") : "" + def dictionaryHash = dictionaryFile != null && dictionaryFile.exists() ? dictionaryFile.text.digest("SHA-256") : "" + return definitionHash + dictionaryHash + } +} + +ext.DictionaryTools = Wrapper diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 888cdede..f4b1c7d5 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -1,7 +1,7 @@ diff --git a/app/src/main/java/io/github/sspanak/tt9/db/entities/CustomWordFile.java b/app/src/main/java/io/github/sspanak/tt9/db/entities/CustomWordFile.java index af1d513e..5f62a97e 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/entities/CustomWordFile.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/entities/CustomWordFile.java @@ -70,7 +70,7 @@ public class CustomWordFile { return null; } - String[] parts = WordFile.splitLine(line); + String[] parts = WordFile.getLineData(line); if (parts == null || parts.length < 2) { return null; } @@ -79,7 +79,7 @@ public class CustomWordFile { } @NonNull public static String getWord(String line) { - String[] parts = WordFile.splitLine(line); + String[] parts = WordFile.getLineData(line); return parts != null && parts.length > 0 ? parts[0] : ""; } } diff --git a/app/src/main/java/io/github/sspanak/tt9/db/entities/WordBatch.java b/app/src/main/java/io/github/sspanak/tt9/db/entities/WordBatch.java index ab363450..b33684e3 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/entities/WordBatch.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/entities/WordBatch.java @@ -4,16 +4,14 @@ import androidx.annotation.NonNull; import java.util.ArrayList; -import io.github.sspanak.tt9.languages.exceptions.InvalidLanguageCharactersException; import io.github.sspanak.tt9.languages.Language; +import io.github.sspanak.tt9.languages.exceptions.InvalidLanguageCharactersException; public class WordBatch { @NonNull private final Language language; @NonNull private final ArrayList words; @NonNull private final ArrayList positions; - private WordPosition lastWordPosition; - public WordBatch(@NonNull Language language, int size) { this.language = language; words = size > 0 ? new ArrayList<>(size) : new ArrayList<>(); @@ -24,31 +22,25 @@ public class WordBatch { this(language, 0); } - public boolean add(@NonNull String word, int frequency, int position) throws InvalidLanguageCharactersException { + public void add(String word, int frequency, int position) throws InvalidLanguageCharactersException { words.add(Word.create(word, frequency, position)); + positions.add(WordPosition.create(language.getDigitSequenceForWord(word), position, position)); + } + + public void add(@NonNull ArrayList words, @NonNull String digitSequence, int position) { + if (words.isEmpty() || digitSequence.isEmpty()) { + return; + } + + for (int i = 0, size = words.size(); i < size; i++) { + this.words.add(Word.create(words.get(i), size - i, position + i)); + } if (position == 0) { - return true; + return; } - String sequence = language.getDigitSequenceForWord(word); - - if (position == 1 || lastWordPosition == null) { - lastWordPosition = WordPosition.create(sequence, position); - } else { - lastWordPosition.end = position; - } - - if (!sequence.equals(lastWordPosition.sequence)) { - lastWordPosition.end--; - positions.add(lastWordPosition); - - lastWordPosition = WordPosition.create(sequence, position); - - return true; - } - - return false; + positions.add(WordPosition.create(digitSequence, position, position + words.size() - 1)); } public void clear() { diff --git a/app/src/main/java/io/github/sspanak/tt9/db/entities/WordFile.java b/app/src/main/java/io/github/sspanak/tt9/db/entities/WordFile.java index 80b06e94..945ff1f4 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/entities/WordFile.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/entities/WordFile.java @@ -4,13 +4,17 @@ import android.content.Context; import android.content.res.AssetManager; import java.io.BufferedReader; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.URL; import java.net.URLConnection; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Locale; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import io.github.sspanak.tt9.R; import io.github.sspanak.tt9.preferences.settings.SettingsStore; @@ -21,26 +25,34 @@ public class WordFile { private final AssetManager assets; private final Context context; - private final String name; + private final String path; + + private int lastCharCode; + private BufferedReader reader; + private String hash = null; private String downloadUrl = null; - private int totalLines = -1; + private int words = -1; private long size = -1; + private int sequences = -1; - public WordFile(Context context, String name, AssetManager assets) { + public WordFile(Context context, String path, AssetManager assets) { this.assets = assets; this.context = context; - this.name = name; + this.path = path; + + lastCharCode = 0; + reader = null; } - public static String[] splitLine(String line) { - String[] parts = { line, "" }; + public static String[] getLineData(String line) { + String[] parts = {line, ""}; // This is faster than String.split() by around 10%, so it's worth having it. // It runs very often, so any other optimizations are welcome. - for (int i = 0 ; i < line.length(); i++) { + for (int i = 0; i < line.length(); i++) { if (line.charAt(i) == ' ') { // the delimiter is TAB parts[0] = line.substring(0, i); parts[1] = i < line.length() - 1 ? line.substring(i + 1) : ""; @@ -52,18 +64,9 @@ public class WordFile { } - public static short getFrequencyFromLineParts(String[] frequencyParts) { - try { - return Short.parseShort(frequencyParts[1]); - } catch (Exception e) { - return 0; - } - } - - public boolean exists() { try { - assets.open(name).close(); + assets.open(path).close(); return true; } catch (IOException e) { return false; @@ -80,8 +83,17 @@ public class WordFile { public BufferedReader getReader() throws IOException { - InputStream stream = exists() ? assets.open(name) : getRemoteStream(); - return new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)); + if (reader != null) { + return reader; + } + + InputStream stream = exists() ? assets.open(path) : getRemoteStream(); + ZipInputStream zipStream = new ZipInputStream(stream); + ZipEntry entry = zipStream.getNextEntry(); + if (entry == null) { + throw new IOException("Dictionary ZIP file: " + path + " is empty."); + } + return reader = new BufferedReader(new InputStreamReader(zipStream, StandardCharsets.UTF_8)); } @@ -99,12 +111,20 @@ public class WordFile { return; } - String revision = rawValue == null || rawValue.isEmpty() ? "" : rawValue; - downloadUrl = revision.isEmpty() ? null : context.getString(R.string.dictionary_url, revision, name); + downloadUrl = null; + String revision = rawValue == null || rawValue.isEmpty() ? "" : rawValue; if (revision.isEmpty()) { - Logger.w(LOG_TAG, "Invalid 'revision' property of: " + name + ". Expecting a string, got: '" + rawValue + "'."); + Logger.w(LOG_TAG, "Invalid 'revision' property of: " + path + ". Expecting a string, got: '" + rawValue + "'."); + return; } + + if (path == null || path.isEmpty()) { + Logger.w(LOG_TAG, "Cannot generate a download URL for an empty path."); + return; + } + + downloadUrl = context.getString(R.string.dictionary_url, revision, new File(path).getName()); } @@ -125,39 +145,62 @@ public class WordFile { hash = rawValue == null || rawValue.isEmpty() ? "" : rawValue; if (hash.isEmpty()) { - Logger.w(LOG_TAG, "Invalid 'hash' property of: " + name + ". Expecting a string, got: '" + rawValue + "'."); + Logger.w(LOG_TAG, "Invalid 'hash' property of: " + path + ". Expecting a string, got: '" + rawValue + "'."); } } - public int getTotalLines() { - if (totalLines < 0) { + public int getSequences() { + if (sequences < 0) { loadProperties(); } - return totalLines; + return sequences; } - public String getFormattedTotalLines(String suffix) { - if (getTotalLines() > 1000000) { - return String.format(Locale.ROOT, "%1.2fM %s", getTotalLines() / 1000000.0, suffix); - } else { - return getTotalLines() / 1000 + "k " + suffix; + private void setSequences(String rawProperty, String rawValue) { + if (!rawProperty.equals("sequences")) { + return; + } + + try { + sequences = Integer.parseInt(rawValue); + } catch (Exception e) { + Logger.w(LOG_TAG, "Invalid 'sequences' property of: " + path + ". Expecting an integer, got: '" + rawValue + "'."); + sequences = 0; } } - private void setTotalLines(String rawProperty, String rawValue) { + public int getWords() { + if (words < 0) { + loadProperties(); + } + + return words; + } + + + public String getFormattedWords(String suffix) { + if (getWords() > 1000000) { + return String.format(Locale.ROOT, "%1.2fM %s", getWords() / 1000000.0, suffix); + } else { + return getWords() / 1000 + "k " + suffix; + } + } + + + private void setWords(String rawProperty, String rawValue) { if (!rawProperty.equals("words")) { return; } try { - totalLines = Integer.parseInt(rawValue); + words = Integer.parseInt(rawValue); } catch (Exception e) { - Logger.w(LOG_TAG, "Invalid 'words' property of: " + name + ". Expecting an integer, got: '" + rawValue + "'."); - totalLines = 0; + Logger.w(LOG_TAG, "Invalid 'words' property of: " + path + ". Expecting an integer, got: '" + rawValue + "'."); + words = 0; } } @@ -184,14 +227,14 @@ public class WordFile { try { size = Long.parseLong(rawValue); } catch (Exception e) { - Logger.w(LOG_TAG, "Invalid 'size' property of: " + name + ". Expecting an integer, got: '" + rawValue + "'."); + Logger.w(LOG_TAG, "Invalid 'size' property of: " + path + ". Expecting an integer, got: '" + rawValue + "'."); size = 0; } } private void loadProperties() { - String propertyFilename = name + ".props.yml"; + String propertyFilename = path.replaceFirst("\\.\\w+$", "") + ".props.yml"; try (BufferedReader reader = new BufferedReader(new InputStreamReader(assets.open(propertyFilename)))) { for (String line; (line = reader.readLine()) != null; ) { @@ -202,11 +245,102 @@ public class WordFile { setDownloadUrl(parts[0], parts[1]); setHash(parts[0], parts[1]); - setTotalLines(parts[0], parts[1]); + setWords(parts[0], parts[1]); + setSequences(parts[0], parts[1]); setSize(parts[0], parts[1]); } } catch (Exception e) { Logger.w(LOG_TAG, "Could not read the property file: " + propertyFilename + ". " + e.getMessage()); } } + + + public String getNextSequence() throws IOException { + if (reader == null || !notEOF()) { + return ""; + } + + StringBuilder sequence = new StringBuilder(); + + // use the last char from getNextWords() if it's a digit + if (Character.isDigit(lastCharCode)) { + sequence.append((char) lastCharCode); + } + + while ((lastCharCode = reader.read()) != -1) { + if (Character.isDigit(lastCharCode)) { + sequence.append((char) lastCharCode); + } else { + break; + } + } + + if (sequence.length() == 0) { + throw new IOException("Could not find next sequence. Unexpected end of file."); + } + + return sequence.toString(); + } + + + public ArrayList getNextWords(String digitSequence) throws IOException { + ArrayList words = new ArrayList<>(); + + if (reader == null || !notEOF()) { + return words; + } + + boolean areWordsSeparated = false; + StringBuilder word = new StringBuilder(); + + // If the word string starts with a space, it means there are words longer than the sequence. + // We must make sure to extract them correctly. + if (lastCharCode == ' ') { + areWordsSeparated = true; + } + // use the last char from getNextSequence() if it's a letter + else if (!Character.isDigit(lastCharCode)) { + word.append((char) lastCharCode); + } + + int sequenceLength = digitSequence.length(); + + // start extracting the words + int wordLength = word.length(); + while ((lastCharCode = reader.read()) != -1) { + if (Character.isDigit(lastCharCode)) { + break; + } + + if (lastCharCode == ' ') { + areWordsSeparated = true; + } else { + word.append((char) lastCharCode); + wordLength++; + } + + if ((areWordsSeparated && lastCharCode == ' ' && wordLength > 0) || (!areWordsSeparated && wordLength == sequenceLength)) { + words.add(word.toString()); + wordLength = 0; + word.setLength(wordLength); + } + } + + if ((areWordsSeparated && wordLength > 0) || (!areWordsSeparated && wordLength == sequenceLength)) { + words.add(word.toString()); + } else if (wordLength > 0) { + throw new IOException("Unexpected end of file. Word: '" + word + "' length (" + wordLength + ") differs from the length of sequence: " + digitSequence); + } + + if (words.isEmpty()) { + throw new IOException("Could not find any words for sequence: " + digitSequence); + } + + return words; + } + + + public boolean notEOF() { + return lastCharCode != -1; + } } diff --git a/app/src/main/java/io/github/sspanak/tt9/db/entities/WordPosition.java b/app/src/main/java/io/github/sspanak/tt9/db/entities/WordPosition.java index 0f37ed27..4a1797fb 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/entities/WordPosition.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/entities/WordPosition.java @@ -7,10 +7,11 @@ public class WordPosition { public int start; public int end; - public static WordPosition create(@NonNull String sequence, int start) { + public static WordPosition create(@NonNull String sequence, int start, int end) { WordPosition position = new WordPosition(); position.sequence = sequence; position.start = start; + position.end = end; return position; } diff --git a/app/src/main/java/io/github/sspanak/tt9/db/exceptions/DictionaryImportException.java b/app/src/main/java/io/github/sspanak/tt9/db/exceptions/DictionaryImportException.java index 390533ac..97ef5d1f 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/exceptions/DictionaryImportException.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/exceptions/DictionaryImportException.java @@ -1,12 +1,10 @@ package io.github.sspanak.tt9.db.exceptions; public class DictionaryImportException extends Exception { - public final String word; public final long line; - public DictionaryImportException(String word, long line) { - super("Dictionary import failed"); - this.word = word; + public DictionaryImportException(String message, long line) { + super(message); this.line = line; } } diff --git a/app/src/main/java/io/github/sspanak/tt9/db/words/DictionaryLoader.java b/app/src/main/java/io/github/sspanak/tt9/db/words/DictionaryLoader.java index eb76bda2..5d3eb148 100644 --- a/app/src/main/java/io/github/sspanak/tt9/db/words/DictionaryLoader.java +++ b/app/src/main/java/io/github/sspanak/tt9/db/words/DictionaryLoader.java @@ -7,6 +7,7 @@ import android.os.Bundle; import android.os.Handler; import java.io.BufferedReader; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.Locale; @@ -79,32 +80,32 @@ public class DictionaryLoader { return true; } - loadThread = new Thread() { - @Override - public void run() { - currentFile = 0; - Timer.start(IMPORT_TIMER); - - sendStartMessage(languages.size()); - - // SQLite does not support parallel queries, so let's import them one by one - for (Language lang : languages) { - if (isInterrupted()) { - break; - } - importAll(context, lang); - currentFile++; - } - - Timer.stop(IMPORT_TIMER); - } - }; - + loadThread = new Thread(() -> loadSync(context, languages)); loadThread.start(); + return true; } + private void loadSync(Context context, ArrayList languages) { + currentFile = 0; + Timer.start(IMPORT_TIMER); + + sendStartMessage(languages.size()); + + // SQLite does not support parallel queries, so let's import them one by one + for (Language lang : languages) { + if (loadThread.isInterrupted()) { + break; + } + importAll(context, lang); + currentFile++; + } + + Timer.stop(IMPORT_TIMER); + } + + public static void load(Context context, Language language) { DictionaryLoadingBar progressBar = DictionaryLoadingBar.getInstance(context); getInstance(context).setOnStatusChange(status -> progressBar.show(context, status)); @@ -210,13 +211,11 @@ public class DictionaryLoader { } catch (DictionaryImportException e) { stop(); sqlite.failTransaction(); - sendImportError(DictionaryImportException.class.getSimpleName(), language.getId(), e.line, e.word); + sendImportError(DictionaryImportException.class.getSimpleName(), language.getId(), e.line); Logger.e( LOG_TAG, - " Invalid word: '" + e.word - + "' in dictionary: '" + language.getDictionaryFile() + "'" - + " on line " + e.line + " Invalid word in dictionary: '" + language.getDictionaryFile() + "'" + " of language '" + language.getName() + "'. " + e.getMessage() ); @@ -256,34 +255,32 @@ public class DictionaryLoader { private void importWordFile(Context context, Language language, int positionShift, float minProgress, float maxProgress) throws Exception { WordFile wordFile = new WordFile(context, language.getDictionaryFile(), assets); - WordBatch batch = new WordBatch(language, wordFile.getTotalLines()); - int currentLine = 1; - float progressRatio = (maxProgress - minProgress) / wordFile.getTotalLines(); + WordBatch batch = new WordBatch(language, SettingsStore.DICTIONARY_IMPORT_BATCH_SIZE + 1); + float progressRatio = (maxProgress - minProgress) / wordFile.getWords(); + int wordCount = 0; - try (BufferedReader br = wordFile.getReader()) { - for (String line; (line = br.readLine()) != null; currentLine++) { + try (BufferedReader ignored = wordFile.getReader()) { + while (wordFile.notEOF()) { if (loadThread.isInterrupted()) { sendProgressMessage(language, 0, 0); throw new DictionaryImportAbortedException(); } - String[] parts = WordFile.splitLine(line); - String word = parts[0]; - short frequency = WordFile.getFrequencyFromLineParts(parts); - try { - boolean isFinalized = batch.add(word, frequency, currentLine + positionShift); - if (isFinalized && batch.getWords().size() > SettingsStore.DICTIONARY_IMPORT_BATCH_SIZE) { + String digitSequence = wordFile.getNextSequence(); + ArrayList words = wordFile.getNextWords(digitSequence); + batch.add(words, digitSequence, wordCount + positionShift); + wordCount += words.size(); + + if (batch.getWords().size() > SettingsStore.DICTIONARY_IMPORT_BATCH_SIZE) { saveWordBatch(batch); batch.clear(); } - } catch (InvalidLanguageCharactersException e) { - throw new DictionaryImportException(word, currentLine); + } catch (IOException e) { + throw new DictionaryImportException(e.getMessage(), wordCount); } - if (wordFile.getTotalLines() > 0) { - sendProgressMessage(language, minProgress + progressRatio * currentLine, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME); - } + sendProgressMessage(language, minProgress + progressRatio * wordCount, SettingsStore.DICTIONARY_IMPORT_PROGRESS_UPDATE_TIME); } } @@ -353,7 +350,7 @@ public class DictionaryLoader { } - private void sendImportError(String message, int langId, long fileLine, String word) { + private void sendImportError(String message, int langId, long fileLine) { if (onStatusChange == null) { Logger.w(LOG_TAG, "Cannot send an import error without a status Handler. Ignoring message."); return; @@ -363,14 +360,13 @@ public class DictionaryLoader { errorMsg.putString("error", message); errorMsg.putLong("fileLine", fileLine + 1); errorMsg.putInt("languageId", langId); - errorMsg.putString("word", word); asyncHandler.post(() -> onStatusChange.accept(errorMsg)); } private void logLoadingStep(String message, Language language, long time) { if (Logger.isDebugLevel()) { - Logger.d(LOG_TAG, message + " for language '" + language.getName() + "' in: " + time + " ms"); + Logger.d(LOG_TAG, message + " for language '" + language.getName() + "' (" + language.getId() + ") in: " + time + " ms"); } } } diff --git a/app/src/main/java/io/github/sspanak/tt9/languages/LanguageDefinition.java b/app/src/main/java/io/github/sspanak/tt9/languages/LanguageDefinition.java index 8a8ce4c1..c3bc3deb 100644 --- a/app/src/main/java/io/github/sspanak/tt9/languages/LanguageDefinition.java +++ b/app/src/main/java/io/github/sspanak/tt9/languages/LanguageDefinition.java @@ -12,6 +12,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import io.github.sspanak.tt9.BuildConfig; import io.github.sspanak.tt9.util.Logger; public class LanguageDefinition { @@ -93,6 +94,9 @@ public class LanguageDefinition { definition.locale = getPropertyFromYaml(yaml, "locale", definition.locale); definition.name = getPropertyFromYaml(yaml, "name", definition.name); + if (definition.dictionaryFile != null) { + definition.dictionaryFile = definition.dictionaryFile.replaceFirst("\\.\\w+$", "." + BuildConfig.DICTIONARY_EXTENSION); + } return definition; } diff --git a/app/src/main/java/io/github/sspanak/tt9/preferences/screens/languageSelection/PreferenceSwitchLanguage.java b/app/src/main/java/io/github/sspanak/tt9/preferences/screens/languageSelection/PreferenceSwitchLanguage.java index 0ceefbc5..cccdca3f 100644 --- a/app/src/main/java/io/github/sspanak/tt9/preferences/screens/languageSelection/PreferenceSwitchLanguage.java +++ b/app/src/main/java/io/github/sspanak/tt9/preferences/screens/languageSelection/PreferenceSwitchLanguage.java @@ -55,7 +55,7 @@ public class PreferenceSwitchLanguage extends SwitchPreferenceCompat { summary .append(", ") .append( - wordFile.getFormattedTotalLines(activity.getString(R.string.language_selection_words)) + wordFile.getFormattedWords(activity.getString(R.string.language_selection_words)) ); // download size diff --git a/app/src/main/java/io/github/sspanak/tt9/ui/notifications/DictionaryLoadingBar.java b/app/src/main/java/io/github/sspanak/tt9/ui/notifications/DictionaryLoadingBar.java index 6d901da3..aae354f3 100644 --- a/app/src/main/java/io/github/sspanak/tt9/ui/notifications/DictionaryLoadingBar.java +++ b/app/src/main/java/io/github/sspanak/tt9/ui/notifications/DictionaryLoadingBar.java @@ -75,8 +75,7 @@ public class DictionaryLoadingBar extends DictionaryProgressNotification { context, error, data.getInt("languageId", -1), - data.getLong("fileLine", -1), - data.getString("word", "") + data.getLong("fileLine", -1) ); } else if (progress >= 0) { hasFailed = false; @@ -133,13 +132,13 @@ public class DictionaryLoadingBar extends DictionaryProgressNotification { } - private void showError(Context context, String errorType, int langId, long line, String word) { + private void showError(Context context, String errorType, int langId, long line) { Language lang = LanguageCollection.getLanguage(context, langId); if (lang == null || errorType.equals(InvalidLanguageException.class.getSimpleName())) { message = resources.getString(R.string.add_word_invalid_language); } else if (errorType.equals(DictionaryImportException.class.getSimpleName()) || errorType.equals(InvalidLanguageCharactersException.class.getSimpleName())) { - message = resources.getString(R.string.dictionary_load_bad_char, word, line, lang.getName()); + message = resources.getString(R.string.dictionary_load_bad_char, line, lang.getName()); } else if (errorType.equals(UnknownHostException.class.getSimpleName()) || errorType.equals(SocketException.class.getSimpleName())) { message = resources.getString(R.string.dictionary_load_no_internet, lang.getName()); } else if (errorType.equals(IOException.class.getSimpleName()) || errorType.equals(FileNotFoundException.class.getSimpleName())) { diff --git a/app/src/main/res/values-bg/strings.xml b/app/src/main/res/values-bg/strings.xml index d0d887c8..7eb3ff5b 100644 --- a/app/src/main/res/values-bg/strings.xml +++ b/app/src/main/res/values-bg/strings.xml @@ -31,7 +31,7 @@ Икона за състояние Показвай икона, когато въвеждането с клавиатура е активно. Отмени зареждането - Неуспешно зареждане. Невалидна дума „%1$s“ на ред %2$d за език „%3$s“. + Неуспешно зареждане. Невалидна дума на ред %1$d за език „%2$s“. Несупешно зареждане на речник за език „%1$s“ (%2$s). Зареждането на речник приключи. Зареждане на речник (%1$s)… diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index b3dd100b..36b9986b 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -47,7 +47,7 @@ Die Reihenfolge der Tasten umkehren Aktivieren Sie, wenn die Tastatur in der ersten Zeile 7–8–9 anstelle von 1–2–3 hat. Laden abbrechen - Laden fehlgeschlagen. Ungültiges Wort „%1$s“ in Zeile %2$d der Sprache „%3$s“. + Laden fehlgeschlagen. Ungültiges Wort in Zeile %1$d der Sprache „%2$s“. Fehler beim Laden des Wörterbuchs für die Sprache „%1$s“ (%2$s). Fehler beim Herunterladen des Wörterbuchs für die Sprache „%1$s“. Überprüfen Sie die Internetverbindung. Laden abgebrochen. diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index 5bce932f..ff18cb8f 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -68,7 +68,7 @@ Espacio automático Insertar un espacio automático después de palabras y signos de puntuación. Carácter cuando se presiona \"0\" dos veces - Error al cargar. Palabra inválida \"%1$s\" en la línea %2$d del idioma \"%3$s\". + Error al cargar. Palabra inválida en la línea %1$d del idioma \"%2$s\". Error al cargar el diccionario para el idioma \"%1$s\" (%2$s). Error al descargar el diccionario para el idioma \"%1$s\". Verifique la conexión a Internet. Carga del diccionario cancelada. diff --git a/app/src/main/res/values-fr/strings.xml b/app/src/main/res/values-fr/strings.xml index 95e7b05c..ee2ed0a2 100644 --- a/app/src/main/res/values-fr/strings.xml +++ b/app/src/main/res/values-fr/strings.xml @@ -44,7 +44,7 @@ Supprimer des mots ajoutés Compatibilité Apparance - Echec du chargement. Mot inadmissible «%1$s» à la ligne %2$d de langue «%3$s». + Echec du chargement. Mot inadmissible à la ligne %1$d de langue «%2$s». Le dictionaire est supprimé avec succès. Envoyer avec «OK» dans Facebook Messenger Toujours au premier plan diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index 6c1cc140..8142e16a 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -61,7 +61,7 @@ Scrittura facilitata Scorciatoie da tastiera Caricamento del dizionario - Caricamento non riuscito. Parola non valida \"%1$s\" alla riga %2$d della lingua \"%3$s\". + Caricamento non riuscito. Parola non valida alla riga %1$d della lingua \"%2$s\". Caricamento del dizionario per la lingua \"%1$s\" non riuscito (%2$s). "Download del dizionario per la lingua \"%1$s\" non riuscito. Controlla la connessione Internet. " Caricamento annullato. diff --git a/app/src/main/res/values-iw/strings.xml b/app/src/main/res/values-iw/strings.xml index 89854d67..a667ccf6 100644 --- a/app/src/main/res/values-iw/strings.xml +++ b/app/src/main/res/values-iw/strings.xml @@ -60,7 +60,7 @@ להפוך את סדר המקשים הפעל את ההגדרה אם המקלדת כוללת את המספרים 7-8-9 בשורה הראשונה, במקום 1-2-3. ביטול טעינה - הטעינה נכשלה. מילה לא חוקית \"%1$s\" בשורה %2$d עבור \"%3$s\". + הטעינה נכשלה. מילה לא חוקית בשורה %1$d עבור \"%2$s\". נכשלה טעינת המילון עבור \"%1$s\" (%2$s). נכשל בהורדת המילון עבור השפה \"%1$s\". בדוק את חיבור האינטרנט. טעינת המילון בוטלה diff --git a/app/src/main/res/values-lt/strings.xml b/app/src/main/res/values-lt/strings.xml index 65e8dc74..c30fc05b 100644 --- a/app/src/main/res/values-lt/strings.xml +++ b/app/src/main/res/values-lt/strings.xml @@ -64,7 +64,7 @@ Būsenos piktograma Rodyti piktogramą, kai aktyvus klaviatūros įvedimas Atšaukti įkėlimą - Įkelti \"%3$s\" kalbos nepavyko. Klaida %2$d eilutėje, netinkamas žodis - \"%1$s\". + Įkelti \"%2$s\" kalbos nepavyko. Klaida %1$d eilutėje, netinkamas žodis. Klaida įkeliant \"%1$s\" kalbos žodyną (%2$s). Nepavyko atsisiųsti žodyno kalbai „%1$s“. Patikrinkite interneto ryšį. Žodyno įkėlimas atšauktas. diff --git a/app/src/main/res/values-nl/strings.xml b/app/src/main/res/values-nl/strings.xml index 688ceb54..28a53a40 100644 --- a/app/src/main/res/values-nl/strings.xml +++ b/app/src/main/res/values-nl/strings.xml @@ -46,7 +46,7 @@ De volgorde van de toetsen omkeren Activeer als het toetsenbord 7–8–9 op de eerste rij heeft, in plaats van 1–2–3. Laden annuleren - Laden mislukt. Ongeldig woord \"%1$s\" op regel %2$d van taal \"%3$s\". + Laden mislukt. Ongeldig woord op regel %1$d van taal \"%2$s\". Het laden van het woordenboek voor de taal \"%1$s\" is mislukt (%2$s). Het downloaden van het woordenboek voor de taal \"%1$s\" is mislukt. Controleer de internetverbinding. Laden geannuleerd. diff --git a/app/src/main/res/values-pt-rBR/strings.xml b/app/src/main/res/values-pt-rBR/strings.xml index 668bbbe9..cb9dca05 100644 --- a/app/src/main/res/values-pt-rBR/strings.xml +++ b/app/src/main/res/values-pt-rBR/strings.xml @@ -59,7 +59,7 @@ Ícone de status Mostrar um ícone quando a digitação estiver ativa. Cancelar Carregamento - Falha no carregamento. \"%1$s\" na linha %2$d do idioma \"%3$s\". + Falha no carregamento. Palavra inválida na linha %1$d do idioma \"%2$s\". Falha no carregamento do dicionário para o idioma \"%1$s\" (%2$s). Falha ao baixar o dicionário para o idioma \"%1$s\". Verifique a conexão com a Internet. Carregamento de dicionário cancelado. diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index 70ed01bc..466f2833 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -67,7 +67,7 @@ Автоматически начинать предложение с заглавной буквы. Символ при двойном нажатии клавиши 0 Отправка с «ОК» в Messenger - Не удалось загрузить словарь. Проблема в слове «%1$s» в строке %2$d для языка «%3$s». + Не удалось загрузить словарь. Проблема в слове в строке %1$d для языка «%2$s». Стереть Уведомления словаря Получать уведомления о обновлениях словаря и о процессе загрузки. diff --git a/app/src/main/res/values-tr/strings.xml b/app/src/main/res/values-tr/strings.xml index 6642827b..599bf88f 100644 --- a/app/src/main/res/values-tr/strings.xml +++ b/app/src/main/res/values-tr/strings.xml @@ -46,7 +46,7 @@ Durum Klavye girişi etkin olduğunda bir simge göster. Yüklemeyi İptal Et - Yükleme başarısız. \"%1$s\" sözcüğü \"%3$s\" dilinin %2$d satırında geçersiz. + Yükleme başarısız. \"%2$s\" dilinde %1$d. satırda geçersiz kelime. “%1$s” dili için sözlük yüklenemedi (%2$s). “%1$s” dili için sözlük indirilemedi. İnternet bağlantısını kontrol edin. Yükleme iptal edildi. diff --git a/app/src/main/res/values-uk/strings.xml b/app/src/main/res/values-uk/strings.xml index 6640ab09..3d810783 100644 --- a/app/src/main/res/values-uk/strings.xml +++ b/app/src/main/res/values-uk/strings.xml @@ -70,7 +70,7 @@ Іконка статусу Показати іконку, коли активне введення з клавіатури. Скасувати завантаження - Завантаження не вдалося. Невірне слово \"%1$s\" у рядку %2$d мови \"%3$s\". + Завантаження не вдалося. Невірне слово у рядку %1$d мови \"%2$s\". Не вдалося завантажити словник для мови \"%1$s\" (%2$s). Не вдалося завантажити словник для мови \"%1$s\". Перевірте підключення до Інтернету. Завантаження словника скасовано. diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 49ad4a11..7686bcc7 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1,6 +1,6 @@ - https://raw.githubusercontent.com/sspanak/tt9/%1$s/app/%2$s + https://raw.githubusercontent.com/sspanak/tt9/%1$s/downloads/%2$s Traditional T9 TT9 TT9 Settings @@ -90,7 +90,7 @@ Enable if the keypad has 7–8–9 on the first row, instead of 1–2–3. Cancel Loading - Loading failed. Invalid word \"%1$s\" on line %2$d of language \"%3$s\". + Loading failed. Invalid word on line %1$d of language \"%2$s\". Failed loading the dictionary for language \"%1$s\" (%2$s). Failed downloading the dictionary for language \"%1$s\". Check the Internet connection. Dictionary load cancelled. diff --git a/app/validate-languages.gradle b/app/validate-languages.gradle index cdbd9afb..39561f3a 100644 --- a/app/validate-languages.gradle +++ b/app/validate-languages.gradle @@ -1,241 +1,311 @@ -static def validateDictionaryWord(String word, int lineNumber, String validCharacters, String errorMsgPrefix) { - int errorCount = 0 - def errors = '' - - if (word.matches("(\\d.+?|.+?\\d|\\d)")) { - errorCount++ - errors += "${errorMsgPrefix}. Found numbers on line ${lineNumber}. Remove all numbers.\n" - } - - if (word.matches("^\\P{L}+\$")) { - errorCount++ - errors += "${errorMsgPrefix}. Found a garbage word: '${word}' on line ${lineNumber}.\n" - } - - if (word.matches("^(.|\\p{L}\\p{M}?)\$")) { - errorCount++ - errors += "${errorMsgPrefix}. Found a single letter: '${word}' on line ${lineNumber}. Only uppercase single letters are allowed. The rest of the alphabet will be added automatically.\n" - } - - if (errorCount == 0 && !word.matches(validCharacters)) { - errorCount++ - errors += "${errorMsgPrefix}. Word '${word}' on line ${lineNumber} contains characters outside of the defined alphabet: $validCharacters.\n" - } - - return [errorCount, errors] -} - - -static def validateDictionaryLine(String line, int lineNumber) { - if (line == "") { - return "There is no word on line ${lineNumber}. Remove all empty lines." - } else if (line.contains(" ")) { - return "Found space on line ${lineNumber}. Make sure each word is on a new line. Phrases are not allowed." - } - - return '' -} - - -static def extractAlphabetCharsFromLine(String line) { - if (line.contains('PUNCTUATION') || line.contains('SPECIAL') || !line.matches('\\s+- \\[.+?\\].*')) { - return '' - } - - return line.replaceFirst('^\\s+- \\[', '').replaceFirst('\\].*', '').replace(',', '').replace(' ', '') -} - - -static def parseLanguageFile(File languageFile, String dictionariesDir) { - String alphabet = "" - File dictionaryFile - int errorCount = 0 - String errorMsg = "" - - boolean hasLayout = false - boolean isLocaleValid = false - String localeString = "" - String dictionaryFileName = "" - - alphabet = languageFile.name.contains("Catalan") ? '·' : alphabet - alphabet = languageFile.name.contains("Hebrew") || languageFile.name.contains("Yiddish") ? '"' : alphabet - - for (String line : languageFile.readLines()) { - if ( - line.matches("^[a-zA-Z].*") - && !line.startsWith("abcString") - && !line.startsWith("dictionaryFile") - && !line.startsWith("hasSpaceBetweenWords") - && !line.startsWith("hasUpperCase") - && !line.startsWith("layout") - && !line.startsWith("locale") - && !line.startsWith("name") - ) { - def parts = line.split(":") - def property = parts.length > 0 ? parts[0] : line - - errorCount++ - errorMsg += "Language '${languageFile.name}' is invalid. Found unknown property: '${property}'.\n" - } - - if ( - (line.startsWith("hasUpperCase") || line.startsWith("hasSpaceBetweenWords")) - && !line.endsWith("yes") && !line.endsWith("no") - ) { - def property = line.replaceAll(":.*\$", "") - def invalidVal = line.replace("hasUpperCase:", "").trim() - errorCount++ - errorMsg += "Language '${languageFile.name}' is invalid. Unrecognized '${property}' value: '${invalidVal}'. Only 'yes' and 'no' are allowed.\n" - } - - if (line.startsWith("layout")) { - hasLayout = true - } - - if (line.startsWith("locale")) { - localeString = line.replace("locale:", "").trim() - isLocaleValid = line.matches("^locale:\\s*[a-z]{2}(?:-[A-Z]{2})?") - } - - if (line.startsWith("dictionaryFile")) { - dictionaryFileName = line.replace("dictionaryFile:", "").trim() - } - - def lineCharacters = extractAlphabetCharsFromLine(line) - alphabet += lineCharacters - } - - if (!hasLayout) { - errorCount++ - errorMsg += "Language '${languageFile.name}' is invalid. Missing 'layout' property.\n" - } - - if (alphabet.isEmpty()) { - errorCount++ - errorMsg += "Language '${languageFile.name}' is invalid. No language characters found. Make sure 'layout' contains series of characters per each key in the format: ' - [a, b, c]' and so on\n" - } - - if (!isLocaleValid) { - errorCount++ - def msg = localeString.isEmpty() ? "Missing 'locale' property." : "Unrecognized locale format: '${localeString}'" - errorMsg += "Language '${languageFile.name}' is invalid. ${msg}\n" - } - - dictionaryFile = new File("$dictionariesDir/${dictionaryFileName}") - if (dictionaryFileName.isEmpty() || !dictionaryFile.exists()) { - errorCount++ - errorMsg += "Could not find dictionary file: '${dictionaryFileName}' in: '${dictionariesDir}'. Make sure 'dictionaryFile' is set correctly in: '${languageFile.name}'.\n" - } - - String[] localeParts = localeString.split(("[-_]")) - Locale locale = new Locale(localeParts[0], localeParts.length > 1 ? localeParts[1] : "") - - return [alphabet, locale, dictionaryFile, errorCount, errorMsg] -} - - -static def parseDictionaryFile(String alphabet, Locale locale, File dictionaryFile, int MAX_ERRORS, String CSV_DELIMITER, int MAX_WORD_FREQUENCY) { - final GEOGRAPHICAL_NAME = ~"[A-Z]\\w+-[^\\n]+" - final VALID_CHARS = alphabet.toUpperCase(locale) == alphabet ? "^[${alphabet}\\-\\.']+\$" : "^[${alphabet}${alphabet.toUpperCase(locale)}\\-\\.']+\$" - final int MAX_SORTING_ERRORS = Math.ceil(MAX_ERRORS / 10) - - def uniqueWords = [:] - - int errorCount = 0 - int sortingErrorCount = 0 - String errorMsg = "" - - def fileContents = dictionaryFile.readLines() - for (int lineNumber = 1, previousWordLength = 0; lineNumber <= fileContents.size() && errorCount < MAX_ERRORS; lineNumber++) { - String line = fileContents.get(lineNumber - 1) - - String error = validateDictionaryLine(line, lineNumber) - if (!error.isEmpty()) { - errorCount++ - errorMsg += "Dictionary '${dictionaryFile.name}' is invalid. ${error}\n" - break - } - - String[] parts = line.split(CSV_DELIMITER, 2) - String word = parts[0] - int frequency - try { - frequency = (parts.length > 1 ? parts[1] : "0") as int - } catch (Exception ignored) { - frequency = -1 - } - - if (frequency < 0 || frequency > MAX_WORD_FREQUENCY) { - errorCount++ - errorMsg += "Dictionary '${dictionaryFile.name}' is invalid. Found out-of-range word frequency: '${parts[1]}' on line ${lineNumber}. Frequency must be an integer between 0 and ${MAX_WORD_FREQUENCY}.\n" - } - - if (sortingErrorCount < MAX_SORTING_ERRORS && word.length() < previousWordLength) { - sortingErrorCount++ - errorCount++ - - if (sortingErrorCount == MAX_SORTING_ERRORS) { - errorMsg += "Too many sorting errors in '${dictionaryFile.name}'. Disabling sorting check until the end of the file.\n" - } else { - errorMsg += "Dictionary '${dictionaryFile.name}' is not sorted. Word: '${word}' on line ${lineNumber} is shorter than the previous one. Ensure all words are sorted by length and sequence.\n" - } - } - previousWordLength = word.length() - - def (wordErrorCount, wordErrors) = validateDictionaryWord(word, lineNumber, VALID_CHARS, "Dictionary '${dictionaryFile.name}' is invalid") - errorCount += wordErrorCount - errorMsg += wordErrors - - String uniqueWordKey = word ==~ GEOGRAPHICAL_NAME ? word : word.toLowerCase(locale) - if (uniqueWords[uniqueWordKey] != null && uniqueWords[uniqueWordKey] == true) { - errorCount++ - errorMsg += "Dictionary '${dictionaryFile.name}' is invalid. Found a repeating word: '${word}' on line ${lineNumber}. Ensure all words appear only once.\n" - } else { - uniqueWords[uniqueWordKey] = true - } - } - - return [errorMsg, errorCount] -} +apply from: 'dictionary-tools.gradle' ext.validateLanguageFiles = { definitionsDir, dictionariesDir, validationDir -> - int errorCount = 0 + int errorCount = 0 - def errorStream = fileTree(definitionsDir).getFiles().parallelStream().map { File languageFile -> - def contentHash = languageFile.text.digest("SHA-1") - def outputFile = new File("${validationDir}/${languageFile.name.replace(".yml", "")}.txt") + def errorStream = fileTree(dir: definitionsDir).getFiles().parallelStream().map { definition -> + if (errorCount >= MAX_ERRORS) { + return "Too many errors! Skipping: ${definition}\n" + } - if (outputFile.exists() && outputFile.text == "${contentHash} OK") { - return "" - } + def (alphabet, sounds, isAlphabeticLanguage, locale, dictionaryFile, langFileErrorCount, langFileErrorMsg) = parseLanguageDefintion(definition, dictionariesDir) - outputFile.text = "" + def languageHash = DictionaryTools.getLanguageHash(definition, dictionaryFile) + def validationFile = new File("${validationDir}/${definition.name.replace(".yml", "")}.txt") - if (errorCount >= MAX_ERRORS) { - return "Too many errors! Skipping: ${languageFile}\n" - } + errorCount += langFileErrorCount + if (!langFileErrorMsg.isEmpty()) { + validationFile.text = "${languageHash} INVALID" + return langFileErrorMsg + } - def (alphabet, locale, dictionaryFile, langFileErrorCount, langFileErrorMsg) = parseLanguageFile(languageFile, dictionariesDir) - errorCount += langFileErrorCount - if (!langFileErrorMsg.isEmpty()) { - outputFile.text += "${contentHash} INVALID" - return langFileErrorMsg - } + if (validationFile.exists() && validationFile.text == "${languageHash} OK") { + return "" + } - def (dictionaryErrorMsg, dictionaryErrorCount) = parseDictionaryFile(alphabet, locale, dictionaryFile, MAX_ERRORS, CSV_DELIMITER, MAX_WORD_FREQUENCY) - errorCount += dictionaryErrorCount - if (!dictionaryErrorMsg.isEmpty()) { - outputFile.text += "${contentHash} INVALID" - return dictionaryErrorMsg - } + def (dictionaryErrorCount, dictionaryErrorMesages) = validateDictionary(dictionaryFile, alphabet, sounds, isAlphabeticLanguage, locale, MAX_ERRORS, CSV_DELIMITER, MAX_WORD_FREQUENCY) + errorCount += dictionaryErrorCount + if (!dictionaryErrorMesages.isEmpty()) { + validationFile.text = "${languageHash} INVALID" + return dictionaryErrorMesages + } - outputFile.text += "${contentHash} OK" - return "" - } + validationFile.text = "${languageHash} OK" + return "" + } - String errorsMsg = errorStream.reduce("", String::concat) - if (errorsMsg) { - throw new GradleException(errorsMsg) - } + String errorsMsg = errorStream.reduce("", String::concat) + if (errorsMsg) { + throw new GradleException(errorsMsg) + } +} + + +ext.parseLanguageDefintion = { File languageFile, String dictionariesDir -> + String alphabet = "" + int layoutKey = 0 + HashMap sounds = new HashMap<>() + HashMap layoutSounds = new HashMap<>() + + File dictionaryFile + int errorCount = 0 + String errorMsg = "" + + boolean hasLayout = false + boolean hasSounds = false + boolean isLocaleValid = false + String localeString = "" + String dictionaryFileName = "" + + alphabet = languageFile.name.contains("Catalan") ? '·' : alphabet + alphabet = languageFile.name.contains("Hebrew") || languageFile.name.contains("Yiddish") ? '"' : alphabet + + for (String line : languageFile.readLines()) { + if ( + line.matches("^[a-zA-Z].*") + && !line.startsWith("abcString") + && !line.startsWith("dictionaryFile") + && !line.startsWith("hasSpaceBetweenWords") + && !line.startsWith("hasUpperCase") + && !line.startsWith("layout") + && !line.startsWith("locale") + && !line.startsWith("name") + && !line.startsWith("sounds") + ) { + def parts = line.split(":") + def property = parts.length > 0 ? parts[0] : line + + errorCount++ + errorMsg += "Language '${languageFile.name}' is invalid. Found unknown property: '${property}'.\n" + } + + if ( + (line.startsWith("hasUpperCase") || line.startsWith("hasSpaceBetweenWords")) + && !line.endsWith("yes") && !line.endsWith("no") + ) { + def property = line.replaceAll(":.*\$", "") + def invalidVal = line.replace("hasUpperCase:", "").trim() + errorCount++ + errorMsg += "Language '${languageFile.name}' is invalid. Unrecognized '${property}' value: '${invalidVal}'. Only 'yes' and 'no' are allowed.\n" + } + + if (line.startsWith("layout")) { + hasLayout = true + } + + if (line.startsWith("sounds")) { + hasSounds = true + } + + if (line.startsWith("locale")) { + localeString = line.replace("locale:", "").trim() + isLocaleValid = line.matches("^locale:\\s*[a-z]{2}(?:-[A-Z]{2})?") + } + + if (line.startsWith("dictionaryFile")) { + dictionaryFileName = line.replace("dictionaryFile:", "").trim() + } + + // alphabet string + def lineCharacters = extractAlphabetCharsFromLine(line) + alphabet += lineCharacters + + // sounds, single letters + if (lineCharacters) { + lineCharacters.each { letter -> + layoutSounds.put(letter, layoutKey.toString()) + } + } else if (line.contains("PUNCTUATION")) { + layoutSounds.put("-", layoutKey.toString()) + layoutSounds.put(".", layoutKey.toString()) + layoutSounds.put("'", layoutKey.toString()) + layoutSounds.put('"', layoutKey.toString()) + layoutSounds.put('·', layoutKey.toString()) + } + + if (isLayoutLine(line)) { + layoutKey++ + } + + // sounds, syllables + def (sound, sequence) = extractSoundFromLine(line) + if (!sound.isEmpty() && !sequence.isEmpty()) { + sounds.put(sound, sequence) + } + } + + if (!hasLayout) { + errorCount++ + errorMsg += "Language '${languageFile.name}' is invalid. Missing 'layout' property.\n" + } + + if (alphabet.isEmpty()) { + errorCount++ + errorMsg += "Language '${languageFile.name}' is invalid. No language characters found. Make sure 'layout' contains series of characters per each key in the format: ' - [a, b, c]' and so on\n" + } + + if (hasSounds && sounds.isEmpty()) { + errorCount++ + errorMsg += "Language '${languageFile.name}' is invalid. 'sounds' property must contain series of phonetic transcriptions per digit sequence in the format: ' - [Yae,1221]' and so on.\n" + } + + if (!isLocaleValid) { + errorCount++ + def msg = localeString.isEmpty() ? "Missing 'locale' property." : "Unrecognized locale format: '${localeString}'" + errorMsg += "Language '${languageFile.name}' is invalid. ${msg}\n" + } + + dictionaryFile = new File("$dictionariesDir/${dictionaryFileName}") + if (dictionaryFileName.isEmpty() || !dictionaryFile.exists()) { + errorCount++ + errorMsg += "Could not find dictionary file: '${dictionaryFileName}' in: '${dictionariesDir}'. Make sure 'dictionaryFile' is set correctly in: '${languageFile.name}'.\n" + } + + String[] localeParts = localeString.split(("[-_]")) + Locale locale = new Locale(localeParts[0], localeParts.length > 1 ? localeParts[1] : "") + + if (!hasSounds && locale != null) { + layoutSounds.forEach { sound, sequence -> + sounds.put(sound.toUpperCase(locale), sequence) + } + } + + return [alphabet, sounds, !hasSounds, locale, dictionaryFile, errorCount, errorMsg] +} + + +// this cannot be static, because DictionaryTools will not be visible +def validateDictionary(File dictionaryFile, String alphabet, HashMap sounds, boolean isAlphabeticLanguage, Locale locale, int maxErrors, String csvDelimiter, int maxWordFrequency) { + final VALID_CHARS = alphabet.toUpperCase(locale) == alphabet ? "^[${alphabet}\\-\\.']+\$" : "^[${alphabet}${alphabet.toUpperCase(locale)}\\-\\.']+\$" + + int errorCount = 0 + String errorMsg = '' + Set uniqueWords = new HashSet<>() + + List fileContents = dictionaryFile.readLines() + for (int lineNumber = 1; lineNumber <= fileContents.size() && errorCount < maxErrors; lineNumber++) { + String line = fileContents.get(lineNumber - 1) + boolean lineHasErrors = false + + String whiteSpaceError = validateNoWhitespace(line, lineNumber) + if (whiteSpaceError) { + lineHasErrors = true + errorCount++ + errorMsg += whiteSpaceError + } + + def (word, transcription, frequency) = DictionaryTools.getDictionaryLineData(line, csvDelimiter) + + String frequencyError = validateFrequency(frequency, maxWordFrequency, dictionaryFile.name, lineNumber) + if (frequencyError) { + lineHasErrors = true + errorCount++ + errorMsg += frequencyError + } + + def (wordErrorCount, wordErrors) = validateWord(word, VALID_CHARS, isAlphabeticLanguage, lineNumber, "Dictionary '${dictionaryFile.name}' is invalid") + if (wordErrorCount > 0) { + errorCount += wordErrorCount + errorMsg += wordErrors + } + + if (uniqueWords.contains(word)) { + lineHasErrors = true + errorCount++ + errorMsg += "Dictionary '${dictionaryFile.name}' is invalid. Found duplicate word: '${word}' on line ${lineNumber}. Remove all duplicates.\n" + } else { + uniqueWords.add(word) + } + + if (lineHasErrors) { + // the validations below make no sense if the previous ones have failed + continue + } + + try { + def transcribedWord = transcription.isEmpty() ? word : transcription + DictionaryTools.wordToDigitSequence(locale, transcribedWord, sounds, !transcription.isEmpty()) + } catch (IllegalArgumentException e) { + errorCount++ + errorMsg += "Dictionary '${dictionaryFile.name}' is invalid. Failed generating digit sequence for word '${word}' on line ${lineNumber}. ${e.message}\n" + } + } + + return [errorCount, errorMsg] +} + +//////////////////// PARSING //////////////////// + +static def extractAlphabetCharsFromLine(String line) { + if (line.contains('PUNCTUATION') || line.contains('SPECIAL') || !isLayoutLine(line)) { + return '' + } + + return line.replaceFirst('^\\s+- \\[', '').replaceFirst('\\].*', '').replace(',', '').replace(' ', '') +} + + +static def extractSoundFromLine(String line) { + if (!line.matches('\\s+- \\[\\w+\\s*,\\s*\\d+\\].*')) { + return ['', ''] + } + + def cleanLine = line.replaceFirst('^\\s+- \\[', '').replaceFirst('\\].*', '').replace(' ', '') + def parts = cleanLine.split(',') + return parts.length > 1 ? [parts[0], parts[1]] : ['', ''] +} + + +static def isLayoutLine(String line) { + return line.matches('\\s+- \\[.+?\\].*') && !line.find('\\d+]') +} + +//////////////////// VALIDATION //////////////////// + +static def validateNoWhitespace(String line, int lineNumber) { + if (line == "") { + return "There is no word on line ${lineNumber}. Remove all empty lines.\n" + } else if (line.contains(" ")) { + return "Found space on line ${lineNumber}. Make sure each word is on a new line. Phrases are not allowed.\n" + } + + return '' +} + + +static def validateFrequency(int frequency, int maxFrequency, String dictionaryFileName, int lineNumber) { + if (frequency < 0 || frequency > maxFrequency) { + return "Dictionary '${dictionaryFileName}' is invalid. Found out-of-range word frequency: '${frequency}' on line ${lineNumber}. Frequency must be an integer between 0 and ${maxFrequency}.\n" + } + + return '' +} + + +static def validateWord(String word, String validCharacters, boolean isAlphabeticLanguage, int lineNumber, String errorMsgPrefix) { + int errorCount = 0 + def errors = '' + + if (word.matches("(\\d.+?|.+?\\d|\\d)")) { + errorCount++ + errors += "${errorMsgPrefix}. Found numbers on line ${lineNumber}. Remove all numbers.\n" + } + + if (word.matches("^\\P{L}+\$")) { + errorCount++ + errors += "${errorMsgPrefix}. Found a garbage word: '${word}' on line ${lineNumber}.\n" + } + + if (isAlphabeticLanguage && word.matches("^(.|\\p{L}\\p{M}?)\$")) { + errorCount++ + errors += "${errorMsgPrefix}. Found a single letter: '${word}' on line ${lineNumber}. Only uppercase single letters are allowed. The rest of the alphabet will be added automatically.\n" + } + + if (errorCount == 0 && isAlphabeticLanguage && !word.matches(validCharacters)) { + errorCount++ + errors += "${errorMsgPrefix}. Word '${word}' on line ${lineNumber} contains characters outside of the defined alphabet: $validCharacters.\n" + } + + return [errorCount, errors] } diff --git a/downloads/ar-utf8.zip b/downloads/ar-utf8.zip new file mode 100644 index 00000000..c96af089 Binary files /dev/null and b/downloads/ar-utf8.zip differ diff --git a/downloads/bg-utf8.zip b/downloads/bg-utf8.zip new file mode 100644 index 00000000..f21ee5ad Binary files /dev/null and b/downloads/bg-utf8.zip differ diff --git a/downloads/ca-utf8.zip b/downloads/ca-utf8.zip new file mode 100644 index 00000000..cddf824b Binary files /dev/null and b/downloads/ca-utf8.zip differ diff --git a/downloads/cz-utf8.zip b/downloads/cz-utf8.zip new file mode 100644 index 00000000..313e892b Binary files /dev/null and b/downloads/cz-utf8.zip differ diff --git a/downloads/da-utf8.zip b/downloads/da-utf8.zip new file mode 100644 index 00000000..9c6db79a Binary files /dev/null and b/downloads/da-utf8.zip differ diff --git a/downloads/de-utf8.zip b/downloads/de-utf8.zip new file mode 100644 index 00000000..213d27c6 Binary files /dev/null and b/downloads/de-utf8.zip differ diff --git a/downloads/en-utf8.zip b/downloads/en-utf8.zip new file mode 100644 index 00000000..a8dc919b Binary files /dev/null and b/downloads/en-utf8.zip differ diff --git a/downloads/es-utf8.zip b/downloads/es-utf8.zip new file mode 100644 index 00000000..da7bfb48 Binary files /dev/null and b/downloads/es-utf8.zip differ diff --git a/downloads/fi-utf8.zip b/downloads/fi-utf8.zip new file mode 100644 index 00000000..119e41be Binary files /dev/null and b/downloads/fi-utf8.zip differ diff --git a/downloads/fr-utf8.zip b/downloads/fr-utf8.zip new file mode 100644 index 00000000..6cf7551a Binary files /dev/null and b/downloads/fr-utf8.zip differ diff --git a/downloads/gr-utf8.zip b/downloads/gr-utf8.zip new file mode 100644 index 00000000..7e7950b5 Binary files /dev/null and b/downloads/gr-utf8.zip differ diff --git a/downloads/he-utf8.zip b/downloads/he-utf8.zip new file mode 100644 index 00000000..95b93bae Binary files /dev/null and b/downloads/he-utf8.zip differ diff --git a/downloads/hing-utf8.zip b/downloads/hing-utf8.zip new file mode 100644 index 00000000..2326f613 Binary files /dev/null and b/downloads/hing-utf8.zip differ diff --git a/downloads/hr-utf8.zip b/downloads/hr-utf8.zip new file mode 100644 index 00000000..21f4b352 Binary files /dev/null and b/downloads/hr-utf8.zip differ diff --git a/downloads/hu-utf8.zip b/downloads/hu-utf8.zip new file mode 100644 index 00000000..14b6f5a8 Binary files /dev/null and b/downloads/hu-utf8.zip differ diff --git a/downloads/id-utf8.zip b/downloads/id-utf8.zip new file mode 100644 index 00000000..d5fe9596 Binary files /dev/null and b/downloads/id-utf8.zip differ diff --git a/downloads/it-utf8.zip b/downloads/it-utf8.zip new file mode 100644 index 00000000..0fbbb973 Binary files /dev/null and b/downloads/it-utf8.zip differ diff --git a/downloads/ji-utf8.zip b/downloads/ji-utf8.zip new file mode 100644 index 00000000..9d700513 Binary files /dev/null and b/downloads/ji-utf8.zip differ diff --git a/downloads/lt-utf8.zip b/downloads/lt-utf8.zip new file mode 100644 index 00000000..fcb95349 Binary files /dev/null and b/downloads/lt-utf8.zip differ diff --git a/downloads/nb-utf8.zip b/downloads/nb-utf8.zip new file mode 100644 index 00000000..e23b6d0d Binary files /dev/null and b/downloads/nb-utf8.zip differ diff --git a/downloads/nl-utf8.zip b/downloads/nl-utf8.zip new file mode 100644 index 00000000..279ec809 Binary files /dev/null and b/downloads/nl-utf8.zip differ diff --git a/downloads/pl-utf8.zip b/downloads/pl-utf8.zip new file mode 100644 index 00000000..86198136 Binary files /dev/null and b/downloads/pl-utf8.zip differ diff --git a/downloads/pt-BR-utf8.zip b/downloads/pt-BR-utf8.zip new file mode 100644 index 00000000..37bedddd Binary files /dev/null and b/downloads/pt-BR-utf8.zip differ diff --git a/downloads/pt-PT-utf8.zip b/downloads/pt-PT-utf8.zip new file mode 100644 index 00000000..9c6cec89 Binary files /dev/null and b/downloads/pt-PT-utf8.zip differ diff --git a/downloads/ro-utf8.zip b/downloads/ro-utf8.zip new file mode 100644 index 00000000..5e9cfb7e Binary files /dev/null and b/downloads/ro-utf8.zip differ diff --git a/downloads/ru-utf8.zip b/downloads/ru-utf8.zip new file mode 100644 index 00000000..92baa92d Binary files /dev/null and b/downloads/ru-utf8.zip differ diff --git a/downloads/sv-utf8.zip b/downloads/sv-utf8.zip new file mode 100644 index 00000000..aac957d6 Binary files /dev/null and b/downloads/sv-utf8.zip differ diff --git a/downloads/sw-utf8.zip b/downloads/sw-utf8.zip new file mode 100644 index 00000000..04ea70ba Binary files /dev/null and b/downloads/sw-utf8.zip differ diff --git a/downloads/th-utf8.zip b/downloads/th-utf8.zip new file mode 100644 index 00000000..9831fdb1 Binary files /dev/null and b/downloads/th-utf8.zip differ diff --git a/downloads/tr-utf8.zip b/downloads/tr-utf8.zip new file mode 100644 index 00000000..d786a7f8 Binary files /dev/null and b/downloads/tr-utf8.zip differ diff --git a/downloads/uk-utf8.zip b/downloads/uk-utf8.zip new file mode 100644 index 00000000..db892c62 Binary files /dev/null and b/downloads/uk-utf8.zip differ diff --git a/downloads/vi-utf8.zip b/downloads/vi-utf8.zip new file mode 100644 index 00000000..0fa69545 Binary files /dev/null and b/downloads/vi-utf8.zip differ diff --git a/gradle.properties b/gradle.properties index 66a32abc..08f5aac8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -6,7 +6,7 @@ # http://www.gradle.org/docs/current/userguide/build_environment.html # Specifies the JVM arguments used for the daemon process. # The setting is particularly useful for tweaking memory settings. -org.gradle.jvmargs=-Xmx4096m -Dfile.encoding=UTF-8 +org.gradle.jvmargs=-Xmx6192m -Dfile.encoding=UTF-8 # When configured, Gradle will run in incubating parallel mode. # This option should only be used with decoupled projects. More details, visit # http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects