wzray/tt9
1
0
Fork 0
Code Activity

optimized the remove-dictionary-repeating-words script

Browse source
This commit is contained in:
sspanak 2024-01-07 18:26:38 +02:00 • committed by Dimo Karaivanov
parent cffa66899d
commit 4b7cef763a
1 changed files with 10 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

33
scripts/remove-dictionary-repeating-words.js
View file

@ -41,48 +41,35 @@ function getRegularWordKey(locale, word) {
function getLowercaseWordKey(locale, word) { function getLowercaseWordKey(locale, word) {
return getWordkey(word).toLocaleLowerCase(locale);
}
function getWordkey(word) {
if (typeof word !== 'string' || word.length === 0) { if (typeof word !== 'string' || word.length === 0) {
return ''; return '';
} }
return word; return word.toLocaleLowerCase(locale);
} }
async function removeRepeatingWords({ fileName, locale }) { async function removeRepeatingWords({ fileName, locale }) {
const wordMap = new Map();
const wordMap = {};
let lineReader = createInterface({ input: createReadStream(fileName) }); let lineReader = createInterface({ input: createReadStream(fileName) });
for await (const line of lineReader) { for await (const line of lineReader) {
wordMap[getLowercaseWordKey(locale, line)] = true; const lowercaseKey = getLowercaseWordKey(locale, line);
} if (lowercaseKey === '') {
lineReader = createInterface({ input: createReadStream(fileName) });
for await (const line of lineReader) {
const word = getWordkey(line);
const lowercaseWord = getLowercaseWordKey(locale, line);
if (word === '') {
continue; continue;
} }
if (!wordMap.has(lowercaseKey)) {
wordMap.set(lowercaseKey, line);
}
if (word !== lowercaseWord) { if (wordMap.has(lowercaseKey) && !wordMap.has(line)) {
delete wordMap[lowercaseWord]; wordMap.set(lowercaseKey, line);
wordMap[word] = true;
} }
} }
return Object.keys(wordMap).sort(); return Array.from(wordMap.values(wordMap)).sort();
} }