wzray/tt9
1
0
Fork 0
Code Activity

optimized the word injestion scripts for speed and allowed running multiple instances

Browse source
This commit is contained in:
sspanak 2024-01-06 13:04:01 +02:00 • committed by Dimo Karaivanov
parent 93e3e0aec4
commit b6b8d5bed0
2 changed files with 35 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

21
scripts/add-new-words.sh
View file

@ -26,14 +26,17 @@ LOCALE=$1
DICTIONARY_FILE=$2
NEW_WORDS_FILE=$3
FREQUENCY_FILE=$4
WORK_DIR="/tmp/TT9_$(uuidgen)"
sed -E 's/[\t0-9]+//g' $DICTIONARY_FILE > /tmp/_TT9_base.txt \
&& node scripts/injest-words.js $NEW_WORDS_FILE > /tmp/_TT9_1.txt \
&& node scripts/remove-foreign-words.js $LOCALE /tmp/_TT9_1.txt $LOCALE /tmp/_TT9_base.txt > /tmp/_TT9_2.txt \
&& cp /tmp/_TT9_base.txt /tmp/_TT9_combined.txt \
&& cat /tmp/_TT9_2.txt >> /tmp/_TT9_combined.txt \
&& node scripts/remove-dictionary-repeating-words.js $LOCALE /tmp/_TT9_combined.txt > /tmp/_TT9_clean.txt \
&& node scripts/inject-dictionary-frequencies.js /tmp/_TT9_clean.txt $FREQUENCY_FILE $LOCALE > /tmp/_TT9_output.txt \
&& cat /tmp/_TT9_output.txt
mkdir -p $WORK_DIR && \
sed -E 's/[\t0-9]+//g' $DICTIONARY_FILE > $WORK_DIR/_TT9_base.txt \
&& node scripts/injest-words.js $NEW_WORDS_FILE > $WORK_DIR/_TT9_1.txt \
&& node scripts/remove-foreign-words.js $LOCALE $WORK_DIR/_TT9_1.txt $LOCALE $WORK_DIR/_TT9_base.txt > $WORK_DIR/_TT9_2.txt \
&& cp $WORK_DIR/_TT9_base.txt $WORK_DIR/_TT9_combined.txt \
&& echo >> $WORK_DIR/_TT9_combined.txt \
&& cat $WORK_DIR/_TT9_2.txt >> $WORK_DIR/_TT9_combined.txt \
&& node scripts/remove-dictionary-repeating-words.js $LOCALE $WORK_DIR/_TT9_combined.txt > $WORK_DIR/_TT9_clean.txt \
&& node scripts/inject-dictionary-frequencies.js $WORK_DIR/_TT9_clean.txt $FREQUENCY_FILE $LOCALE > $WORK_DIR/_TT9_output.txt \
&& cat $WORK_DIR/_TT9_output.txt
rm -f /tmp/_TT9*
rm -rf $WORK_DIR

47
scripts/injest-words.js
View file

@ -43,10 +43,7 @@ function cleanSpecialChars(line) {
}
return line
.replaceAll(/[\x01-\x20]+/g, ' ')
.replaceAll(/[&\s,":;\*/]+/g, ' ')
.replaceAll(/\s+/g, ' ')
.replaceAll(/[\[\]\.\?\(\)]/g, '')
.replace(/[\x01-\x20&",:;*\/\[\].?()]+/g, ' ')
.split(' ')
.filter(w => w.length > 1 && !digits.test(w));
}
@ -57,50 +54,52 @@ function splitDashedWords(inputWords) {
return [];
}
const roots = {};
const words = {};
const wordsSet = new Set();
for (const word of inputWords) {
if (!word.includes('-')) {
words[word] = true;
wordsSet.add(word);
continue;
}
const parts = word.split('-');
let root = '';
for (let i = 0; i < parts.length - 1; i++) {
const key = `${parts[i]}-`;
if (key in roots) {
words[key] = true;
} else {
roots[key] = true;
words[parts[i]] = true;
}
root += `${parts[i]}-`;
wordsSet.add(root);
}
words[parts[parts.length - 1]] = true;
wordsSet.add(parts[parts.length - 1]);
}
return Object.keys(words);
return Array.from(wordsSet);
}
async function work({ fileName }) {
words = [];
const wordsSet = new Set();
const lineReader = createInterface({ input: createReadStream(fileName) });
let lineReader = createInterface({ input: createReadStream(fileName) });
for await (const line of lineReader) {
newWords = cleanSpecialChars(line);
const newWords = cleanSpecialChars(line);
words = [
...words,
...newWords
];
for (let i = 0; i < newWords.length; i++) {
wordsSet.add(newWords[i]);
}
}
return splitDashedWords(words).filter(w => w.length > 1).sort();
const wordsArray = Array.from(wordsSet);
const splitWords = splitDashedWords(wordsArray);
const filteredAndSortedWords = splitWords.filter(word => word.length > 1).sort();
return filteredAndSortedWords;
}
/** main **/
work(validateInput())
.then(words => printWords(words))