wzray/tt9
1
0
Fork 0
Code Activity

optimized the remove-dictionary-repeating-words script

Browse source
This commit is contained in:
sspanak 2024-01-07 18:26:38 +02:00 • committed by Dimo Karaivanov
parent cffa66899d
commit 4b7cef763a
1 changed files with 10 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

33
scripts/remove-dictionary-repeating-words.js
View file

@ -41,48 +41,35 @@ function getRegularWordKey(locale, word) {
function getLowercaseWordKey(locale, word) {
return getWordkey(word).toLocaleLowerCase(locale);
}
function getWordkey(word) {
if (typeof word !== 'string' || word.length === 0) {
return '';
}
return word;
return word.toLocaleLowerCase(locale);
}
async function removeRepeatingWords({ fileName, locale }) {
const wordMap = {};
const wordMap = new Map();
let lineReader = createInterface({ input: createReadStream(fileName) });
for await (const line of lineReader) {
wordMap[getLowercaseWordKey(locale, line)] = true;
}
lineReader = createInterface({ input: createReadStream(fileName) });
for await (const line of lineReader) {
const word = getWordkey(line);
const lowercaseWord = getLowercaseWordKey(locale, line);
if (word === '') {
const lowercaseKey = getLowercaseWordKey(locale, line);
if (lowercaseKey === '') {
continue;
}
if (!wordMap.has(lowercaseKey)) {
wordMap.set(lowercaseKey, line);
}
if (word !== lowercaseWord) {
delete wordMap[lowercaseWord];
wordMap[word] = true;
if (wordMap.has(lowercaseKey) && !wordMap.has(line)) {
wordMap.set(lowercaseKey, line);
}
}
return Object.keys(wordMap).sort();
return Array.from(wordMap.values(wordMap)).sort();
}