wzray/tt9
1
0
Fork 0
Code Activity

optimized the word injestion scripts for speed and allowed running multiple instances

Browse source
This commit is contained in:
sspanak 2024-01-06 13:04:01 +02:00 • committed by Dimo Karaivanov
parent 93e3e0aec4
commit b6b8d5bed0
2 changed files with 35 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

21
scripts/add-new-words.sh
View file

@ -26,14 +26,17 @@ LOCALE=$1
DICTIONARY_FILE=$2 DICTIONARY_FILE=$2
NEW_WORDS_FILE=$3 NEW_WORDS_FILE=$3
FREQUENCY_FILE=$4 FREQUENCY_FILE=$4
WORK_DIR="/tmp/TT9_$(uuidgen)"
sed -E 's/[\t0-9]+//g' $DICTIONARY_FILE > /tmp/_TT9_base.txt \ mkdir -p $WORK_DIR && \
&& node scripts/injest-words.js $NEW_WORDS_FILE > /tmp/_TT9_1.txt \ sed -E 's/[\t0-9]+//g' $DICTIONARY_FILE > $WORK_DIR/_TT9_base.txt \
&& node scripts/remove-foreign-words.js $LOCALE /tmp/_TT9_1.txt $LOCALE /tmp/_TT9_base.txt > /tmp/_TT9_2.txt \ && node scripts/injest-words.js $NEW_WORDS_FILE > $WORK_DIR/_TT9_1.txt \
&& cp /tmp/_TT9_base.txt /tmp/_TT9_combined.txt \ && node scripts/remove-foreign-words.js $LOCALE $WORK_DIR/_TT9_1.txt $LOCALE $WORK_DIR/_TT9_base.txt > $WORK_DIR/_TT9_2.txt \
&& cat /tmp/_TT9_2.txt >> /tmp/_TT9_combined.txt \ && cp $WORK_DIR/_TT9_base.txt $WORK_DIR/_TT9_combined.txt \
&& node scripts/remove-dictionary-repeating-words.js $LOCALE /tmp/_TT9_combined.txt > /tmp/_TT9_clean.txt \ && echo >> $WORK_DIR/_TT9_combined.txt \
&& node scripts/inject-dictionary-frequencies.js /tmp/_TT9_clean.txt $FREQUENCY_FILE $LOCALE > /tmp/_TT9_output.txt \ && cat $WORK_DIR/_TT9_2.txt >> $WORK_DIR/_TT9_combined.txt \
&& cat /tmp/_TT9_output.txt && node scripts/remove-dictionary-repeating-words.js $LOCALE $WORK_DIR/_TT9_combined.txt > $WORK_DIR/_TT9_clean.txt \
&& node scripts/inject-dictionary-frequencies.js $WORK_DIR/_TT9_clean.txt $FREQUENCY_FILE $LOCALE > $WORK_DIR/_TT9_output.txt \
&& cat $WORK_DIR/_TT9_output.txt
rm -f /tmp/_TT9* rm -rf $WORK_DIR

47
scripts/injest-words.js
View file

@ -43,10 +43,7 @@ function cleanSpecialChars(line) {
} }
return line return line
.replaceAll(/[\x01-\x20]+/g, ' ') .replace(/[\x01-\x20&",:;*\/\[\].?()]+/g, ' ')
.replaceAll(/[&\s,":;\*/]+/g, ' ')
.replaceAll(/\s+/g, ' ')
.replaceAll(/[\[\]\.\?\(\)]/g, '')
.split(' ') .split(' ')
.filter(w => w.length > 1 && !digits.test(w)); .filter(w => w.length > 1 && !digits.test(w));
} }
@ -57,50 +54,52 @@ function splitDashedWords(inputWords) {
return []; return [];
} }
const roots = {}; const wordsSet = new Set();
const words = {};
for (const word of inputWords) { for (const word of inputWords) {
if (!word.includes('-')) { if (!word.includes('-')) {
words[word] = true; wordsSet.add(word);
continue; continue;
} }
const parts = word.split('-'); const parts = word.split('-');
let root = '';
for (let i = 0; i < parts.length - 1; i++) { for (let i = 0; i < parts.length - 1; i++) {
const key = `${parts[i]}-`; root += `${parts[i]}-`;
if (key in roots) { wordsSet.add(root);
words[key] = true;
} else {
roots[key] = true;
words[parts[i]] = true;
}
} }
words[parts[parts.length - 1]] = true; wordsSet.add(parts[parts.length - 1]);
} }
return Object.keys(words); return Array.from(wordsSet);
} }
async function work({ fileName }) { async function work({ fileName }) {
words = []; const wordsSet = new Set();
const lineReader = createInterface({ input: createReadStream(fileName) });
let lineReader = createInterface({ input: createReadStream(fileName) });
for await (const line of lineReader) { for await (const line of lineReader) {
newWords = cleanSpecialChars(line); const newWords = cleanSpecialChars(line);
words = [ for (let i = 0; i < newWords.length; i++) {
...words, wordsSet.add(newWords[i]);
...newWords }
];
} }
return splitDashedWords(words).filter(w => w.length > 1).sort(); const wordsArray = Array.from(wordsSet);
const splitWords = splitDashedWords(wordsArray);
const filteredAndSortedWords = splitWords.filter(word => word.length > 1).sort();
return filteredAndSortedWords;
} }
/** main **/ /** main **/
work(validateInput()) work(validateInput())
.then(words => printWords(words)) .then(words => printWords(words))