wzray/tt9
1
0
Fork 0
Code Activity

scripts for cleaning up and adding new lists of words

Browse source
This commit is contained in:
sspanak 2023-12-19 13:02:16 +02:00
parent f91edd8878
commit 2f448636e9
2 changed files with 146 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

39
scripts/add-new-words.sh Executable file
View file

@ -0,0 +1,39 @@
#!/bin/bash
if [ $# -lt 4 ]; then
echo "Usage: $0 LOCALE base-dictionary-file.csv new-words-file.txt frequency-file.csv"
echo 'Cleans up and adds new words to a dictionary file.'
echo 'LOCALE could be any valid JS locale, for exmaple: en, en-US, etc...'
exit 1
fi
if ! [[ -f $2 ]]; then
echo "base-dictionary-file: '$2' does not exist"
exit 2
fi
if ! [[ -f $3 ]]; then
echo "new-words-file: '$3' does not exist"
exit 2
fi
if ! [[ -f $4 ]]; then
echo "frequency-file: '$4' does not exist"
exit 2
fi
LOCALE=$1
DICTIONARY_FILE=$2
NEW_WORDS_FILE=$3
FREQUENCY_FILE=$4
sed -E 's/[\t0-9]+//g' $DICTIONARY_FILE > /tmp/_TT9_base.txt \
&& node scripts/injest-words.js $NEW_WORDS_FILE > /tmp/_TT9_1.txt \
&& node scripts/remove-foreign-words.js $LOCALE /tmp/_TT9_1.txt $LOCALE /tmp/_TT9_base.txt > /tmp/_TT9_2.txt \
&& cp /tmp/_TT9_base.txt /tmp/_TT9_combined.txt \
&& cat /tmp/_TT9_2.txt >> /tmp/_TT9_combined.txt \
&& node scripts/remove-dictionary-repeating-words.js $LOCALE /tmp/_TT9_combined.txt > /tmp/_TT9_clean.txt \
&& node scripts/inject-dictionary-frequencies.js /tmp/_TT9_clean.txt $FREQUENCY_FILE $LOCALE > /tmp/_TT9_output.txt \
&& cat /tmp/_TT9_output.txt
rm -f /tmp/_TT9*