From 4b7cef763ac58fcbf2827ec52216d44287cfa9f9 Mon Sep 17 00:00:00 2001
From: sspanak <doftor.livain@gmail.com>
Date: Sun, 7 Jan 2024 18:26:38 +0200
Subject: [PATCH] optimized the remove-dictionary-repeating-words script

---
 scripts/remove-dictionary-repeating-words.js | 33 ++++++--------------
 1 file changed, 10 insertions(+), 23 deletions(-)

diff --git a/scripts/remove-dictionary-repeating-words.js b/scripts/remove-dictionary-repeating-words.js
index b2305e1c..50e3d8f7 100644
--- a/scripts/remove-dictionary-repeating-words.js
+++ b/scripts/remove-dictionary-repeating-words.js
@@ -41,48 +41,35 @@ function getRegularWordKey(locale, word) {
 
 
 function getLowercaseWordKey(locale, word) {
-	return getWordkey(word).toLocaleLowerCase(locale);
-}
-
-
-
-function getWordkey(word) {
 	if (typeof word !== 'string' || word.length === 0) {
 		return '';
 	}
 
-	return word;
+	return word.toLocaleLowerCase(locale);
 }
 
 
 
 async function removeRepeatingWords({ fileName, locale }) {
-
-	const wordMap = {};
+	const wordMap = new Map();
 
 	let lineReader = createInterface({ input: createReadStream(fileName) });
 	for await (const line of lineReader) {
-		wordMap[getLowercaseWordKey(locale, line)] = true;
-	}
-
-
-	lineReader = createInterface({ input: createReadStream(fileName) });
-	for await (const line of lineReader) {
-		const word = getWordkey(line);
-		const lowercaseWord = getLowercaseWordKey(locale, line);
-
-		if (word === '') {
+		const lowercaseKey = getLowercaseWordKey(locale, line);
+		if (lowercaseKey === '') {
 			continue;
 		}
 
+		if (!wordMap.has(lowercaseKey)) {
+			wordMap.set(lowercaseKey, line);
+		}
 
-		if (word !== lowercaseWord) {
-			delete wordMap[lowercaseWord];
-			wordMap[word] = true;
+		if (wordMap.has(lowercaseKey) && !wordMap.has(line)) {
+			wordMap.set(lowercaseKey, line);
 		}
 	}
 
-	return Object.keys(wordMap).sort();
+	return Array.from(wordMap.values(wordMap)).sort();
 }