wzray/tt9
1
0
Fork 0
Code Activity

dictionary word order is now validated during build

Browse source
This commit is contained in:
sspanak 2024-04-14 11:33:21 +03:00 • committed by Dimo Karaivanov
parent 4422d41918
commit 2418c9c4c3
1 changed files with 41 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

68
app/validate-languages.gradle
View file

@ -126,49 +126,63 @@ static def parseLanguageFile(File languageFile, String dictionariesDir) {
static def parseDictionaryFile(String alphabet, File dictionaryFile, int MAX_ERRORS, String CSV_DELIMITER, int MAX_WORD_FREQUENCY) { static def parseDictionaryFile(String alphabet, File dictionaryFile, int MAX_ERRORS, String CSV_DELIMITER, int MAX_WORD_FREQUENCY) {
final GEOGRAPHICAL_NAME = ~"[A-Z]\\w+-[^\\n]+" final GEOGRAPHICAL_NAME = ~"[A-Z]\\w+-[^\\n]+"
final VALID_CHARS = alphabet.toUpperCase() == alphabet ? "^[${alphabet}\\-\\.']+\$" : "^[${alphabet}${alphabet.toUpperCase()}\\-\\.']+\$" final VALID_CHARS = alphabet.toUpperCase() == alphabet ? "^[${alphabet}\\-\\.']+\$" : "^[${alphabet}${alphabet.toUpperCase()}\\-\\.']+\$"
final int MAX_SORTING_ERRORS = Math.ceil(MAX_ERRORS / 10)
def uniqueWords = [:] def uniqueWords = [:]
int errorCount = 0 int errorCount = 0
int sortingErrorCount = 0
String errorMsg = "" String errorMsg = ""
def fileContents = dictionaryFile.readLines() def fileContents = dictionaryFile.readLines()
for (int lineNumber = 1; lineNumber <= fileContents.size() && errorCount < MAX_ERRORS; lineNumber++) { for (int lineNumber = 1, previousWordLength = 0; lineNumber <= fileContents.size() && errorCount < MAX_ERRORS; lineNumber++) {
String line = fileContents.get(lineNumber - 1) String line = fileContents.get(lineNumber - 1)
String error = validateDictionaryLine(line, lineNumber) String error = validateDictionaryLine(line, lineNumber)
if (!error.isEmpty()) { if (!error.isEmpty()) {
errorCount++ errorCount++
errorMsg += "Dictionary '${dictionaryFile.name}' is invalid. ${error}.\n" errorMsg += "Dictionary '${dictionaryFile.name}' is invalid. ${error}\n"
break break
} }
String[] parts = line.split(CSV_DELIMITER, 2) String[] parts = line.split(CSV_DELIMITER, 2)
String word = parts[0] String word = parts[0]
int frequency int frequency
try { try {
frequency = (parts.length > 1 ? parts[1] : "0") as int frequency = (parts.length > 1 ? parts[1] : "0") as int
} catch (Exception ignored) { } catch (Exception ignored) {
frequency = -1 frequency = -1
} }
if (frequency < 0 || frequency > MAX_WORD_FREQUENCY) { if (frequency < 0 || frequency > MAX_WORD_FREQUENCY) {
errorCount++ errorCount++
errorMsg += "Dictionary '${dictionaryFile.name}' is invalid. Found out-of-range word frequency: '${parts[1]}' on line ${lineNumber}. Frequency must be an integer between 0 and ${MAX_WORD_FREQUENCY}.\n" errorMsg += "Dictionary '${dictionaryFile.name}' is invalid. Found out-of-range word frequency: '${parts[1]}' on line ${lineNumber}. Frequency must be an integer between 0 and ${MAX_WORD_FREQUENCY}.\n"
} }
def (wordErrorCount, wordErrors) = validateDictionaryWord(word, lineNumber, VALID_CHARS, "Dictionary '${dictionaryFile.name}' is invalid") if (sortingErrorCount < MAX_SORTING_ERRORS && word.length() < previousWordLength) {
errorCount += wordErrorCount sortingErrorCount++
errorMsg += wordErrors errorCount++
String uniqueWordKey = word ==~ GEOGRAPHICAL_NAME ? word : word.toLowerCase() if (sortingErrorCount == MAX_SORTING_ERRORS) {
if (uniqueWords[uniqueWordKey] != null && uniqueWords[uniqueWordKey] == true) { errorMsg += "Too many sorting errors in '${dictionaryFile.name}'. Disabling sorting check until the end of the file.\n"
errorCount++
errorMsg += "Dictionary '${dictionaryFile.name}' is invalid. Found a repeating word: '${word}' on line ${lineNumber}. Ensure all words appear only once.\n"
} else { } else {
uniqueWords[uniqueWordKey] = true errorMsg += "Dictionary '${dictionaryFile.name}' is not sorted. Word: '${word}' on line ${lineNumber} is shorter than the previous one. Ensure all words are sorted by length and sequence.\n"
} }
} }
previousWordLength = word.length()
def (wordErrorCount, wordErrors) = validateDictionaryWord(word, lineNumber, VALID_CHARS, "Dictionary '${dictionaryFile.name}' is invalid")
errorCount += wordErrorCount
errorMsg += wordErrors
String uniqueWordKey = word ==~ GEOGRAPHICAL_NAME ? word : word.toLowerCase()
if (uniqueWords[uniqueWordKey] != null && uniqueWords[uniqueWordKey] == true) {
errorCount++
errorMsg += "Dictionary '${dictionaryFile.name}' is invalid. Found a repeating word: '${word}' on line ${lineNumber}. Ensure all words appear only once.\n"
} else {
uniqueWords[uniqueWordKey] = true
}
}
return [errorMsg, errorCount] return [errorMsg, errorCount]
} }