tt9/scripts/remove-dictionary-repeating-words.js

const { basename } = require('path');
const { createReadStream, existsSync } = require('fs');
const { createInterface } = require('readline');

const GEO_NAME = /[A-Z]\w+\-[^\n]+/;


function print(str) {
	process.stdout.write(`${str}\n`);
}


function printHelp() {
	print(`Usage ${basename(process.argv[1])} LOCALE FILENAME.txt [--prefer-lowercase]`);
	print('Removes repeating words from a word list');
	print("If --prefer-lowercase is set, the lowercase variants will be preserved, otherwise capitalized or uppercase variants will remain.")
	print('\nLocale could be any valid JS locale, for example: en, en-US, etc...');
}



function validateInput() {
	if (process.argv.length < 4) {
		printHelp();
		process.exit(1);
	}


	if (!existsSync(process.argv[3])) {
		console.error(`Failure! Could not find file "${process.argv[3]}."`);
		process.exit(2);
	}

	return {
		fileName: process.argv[3],
		locale: process.argv[2],
		preferLowercase: !!process.argv[4]
	};
}



function getRegularWordKey(locale, word) {
	if (typeof word !== 'string' || word.length === 0) {
		return '';
	}

	return GEO_NAME.test(word) ? word : word.toLocaleLowerCase(locale);
}



function getLowercaseWordKey(locale, word) {
	if (typeof word !== 'string' || word.length === 0) {
		return '';
	}

	return word.toLocaleLowerCase(locale);
}



async function removeRepeatingWords({ fileName, locale, preferLowercase }) {
	const wordMap = new Map();

	let lineReader = createInterface({ input: createReadStream(fileName) });
	for await (const line of lineReader) {
		const lowercaseKey = getLowercaseWordKey(locale, line);
		if (lowercaseKey === '') {
			continue;
		}

		if (!wordMap.has(lowercaseKey)) {
			wordMap.set(lowercaseKey, line);
		}

		if (!preferLowercase && wordMap.has(lowercaseKey) && !wordMap.has(line)) {
			wordMap.set(lowercaseKey, line);
		}
	}

	return Array.from(wordMap.values(wordMap)).sort();
}



function printWords(wordList) {
	if (!Array.isArray(wordList)) {
		return;
	}

	wordList.forEach(w => print(w));
}



/** main **/
removeRepeatingWords(validateInput())
	.then(words => printWords(words))
	.catch(e => console.error(e));