added support for preventing dashed words to be broken in the word injestion scripts

scripts/add-new-words.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
 if [ $# -lt 4 ]; then
-	echo "Usage: $0 LOCALE base-dictionary-file.csv new-words-file.txt frequency-file.csv"
+	echo "Usage: $0 LOCALE base-dictionary-file.csv new-words-file.txt frequency-file.csv [ignore-split-list.txt]"
-	echo 'Cleans up and adds new words to a dictionary file.'
+	echo 'Cleans up and adds new words to a dictionary file. Optionally, it could skip splitting the words from "ignore-split-list.txt"'
 	echo 'LOCALE could be any valid JS locale, for exmaple: en, en-US, etc...'
 	exit 1
 fi
@@ -22,15 +22,17 @@ if ! [[ -f $4 ]]; then
 	exit 2
 fi
 
+
 LOCALE=$1
 DICTIONARY_FILE=$2
 NEW_WORDS_FILE=$3
 FREQUENCY_FILE=$4
+IGNORE_SPLIT_LIST_FILE=$5
 WORK_DIR="/tmp/TT9_$(uuidgen)"
 
 mkdir -p $WORK_DIR && \
 sed -E 's/[\t0-9]+//g' $DICTIONARY_FILE > $WORK_DIR/_TT9_base.txt \
-	&& node scripts/injest-words.js $NEW_WORDS_FILE > $WORK_DIR/_TT9_1.txt \
+	&& node scripts/injest-words.js $NEW_WORDS_FILE $IGNORE_SPLIT_LIST_FILE > $WORK_DIR/_TT9_1.txt \
 	&& node scripts/remove-foreign-words.js $LOCALE $WORK_DIR/_TT9_1.txt $LOCALE $WORK_DIR/_TT9_base.txt > $WORK_DIR/_TT9_2.txt \
 	&& cp $WORK_DIR/_TT9_base.txt $WORK_DIR/_TT9_combined.txt \
 	&& echo >> $WORK_DIR/_TT9_combined.txt \

scripts/injest-words.js

@@ -4,8 +4,9 @@ const { createInterface } = require('readline');
 
 
 function printHelp() {
-	console.log(`Usage ${basename(process.argv[1])} word-list.txt`);
+	console.log(`Usage ${basename(process.argv[1])} word-list.txt [split-ignore-list.txt]`);
 	console.log('Breaks dashed words into separate words, puts multiple words on a line on new lines and deletes repeating new lines.');
+	console.log('The split-ignore-list is optional. Allows for not splitting certain words by dashes.');
 }
 
 
@@ -21,9 +22,15 @@ function validateInput() {
 		process.exit(2);
 	}
 
-	return {
+	if (process.argv[3] && !existsSync(process.argv[3])) {
-		fileName: process.argv[2]
+		console.error(`Failure! Could not ignore list file "${process.argv[3]}."`);
+		process.exit(2);
 	}
+
+	return {
+		fileName: process.argv[2],
+		ignoreListFileName: process.argv[3]
+	};
 }
 
 
@@ -49,15 +56,21 @@ function cleanSpecialChars(line) {
 }
 
 
-function splitDashedWords(inputWords) {
+function splitDashedWords(inputWords, ignoreList) {
-	if (!Array.isArray(inputWords)) {
+	const ignoreWords = ignoreList instanceof Set ? ignoreList : new Set();
+	if (!(inputWords instanceof Set)) {
 		return [];
 	}
 
 	const dashedRoots = new Set();
 	const repeatingDashedRoots = new Set();
+	const outputWords = new Set();
 
 	for (const word of inputWords) {
+		if (ignoreWords.has(word)) {
+			continue;
+		}
+
 		const [root, ...others] = word.split('-');
 		if (root === undefined || others.length != 1) {
 			continue;
@@ -70,8 +83,6 @@ function splitDashedWords(inputWords) {
 		}
 	}
 
-	const outputWords = new Set();
-
 	for (const word of inputWords) {
 		const [root, ...others] = word.split('-');
 		if (root && others.length === 1 && repeatingDashedRoots.has(root)) {
@@ -86,22 +97,24 @@ function splitDashedWords(inputWords) {
 }
 
 
+async function readWords(fileName) {
+	const words = new Set();
 
-async function work({ fileName }) {
+	if (!fileName) {
-	const wordsSet = new Set();
+		return words;
-
-	const lineReader = createInterface({ input: createReadStream(fileName) });
-
-	for await (const line of lineReader) {
-		const newWords = cleanSpecialChars(line);
-
-		for (let i = 0; i < newWords.length; i++) {
-			wordsSet.add(newWords[i]);
-		}
 	}
 
-	const wordsArray = Array.from(wordsSet);
+	for await (const line of createInterface({ input: createReadStream(fileName) })) {
-	const splitWords = splitDashedWords(wordsArray);
+		cleanSpecialChars(line).forEach(w => words.add(w));
+	}
+
+	return words;
+}
+
+
+async function work({ fileName, ignoreListFileName }) {
+	const [ words, ignoreList ] = await Promise.all([ readWords(fileName), readWords(ignoreListFileName) ]);
+	const splitWords = splitDashedWords(words, ignoreList);
 	const filteredAndSortedWords = splitWords.filter(word => word.length > 1).sort();
 
 	return filteredAndSortedWords;