wzray/tt9
1
0
Fork 0
Code Activity

improved dictionary processing scripts

Browse source 
* remove-repeating-words script now supports lowercase/uppercase option

  * remove-foreign-words script now supports foreign words file of more than 2^24 words
This commit is contained in:
sspanak 2024-03-12 16:28:37 +02:00 • committed by Dimo Karaivanov
parent c0c95dedb9
commit 18aef27996
2 changed files with 37 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

24
scripts/remove-dictionary-repeating-words.js
View file

@ -5,10 +5,16 @@ const { createInterface } = require('readline');
const GEO_NAME = /[A-Z]\w+\-[^\n]+/; const GEO_NAME = /[A-Z]\w+\-[^\n]+/;
function print(str) {
process.stdout.write(`${str}\n`);
}
function printHelp() { function printHelp() {
console.log(`Usage ${basename(process.argv[1])} LOCALE FILENAME.txt `); print(`Usage ${basename(process.argv[1])} LOCALE FILENAME.txt [--prefer-lowercase]`);
console.log('Removes repeating words from a word list'); print('Removes repeating words from a word list');
console.log('\nLocale could be any valid JS locale, for exmaple: en, en-US, etc...'); print("If --prefer-lowercase is set, the lowercase variants will be preserved, otherwise capitalized or uppercase variants will remain.")
print('\nLocale could be any valid JS locale, for example: en, en-US, etc...');
} }
@ -25,7 +31,11 @@ function validateInput() {
process.exit(2); process.exit(2);
} }
return { fileName: process.argv[3], locale: process.argv[2] }; return {
fileName: process.argv[3],
locale: process.argv[2],
preferLowercase: !!process.argv[4]
};
} }
@ -50,7 +60,7 @@ function getLowercaseWordKey(locale, word) {
async function removeRepeatingWords({ fileName, locale }) { async function removeRepeatingWords({ fileName, locale, preferLowercase }) {
const wordMap = new Map(); const wordMap = new Map();
let lineReader = createInterface({ input: createReadStream(fileName) }); let lineReader = createInterface({ input: createReadStream(fileName) });
@ -64,7 +74,7 @@ async function removeRepeatingWords({ fileName, locale }) {
wordMap.set(lowercaseKey, line); wordMap.set(lowercaseKey, line);
} }
if (wordMap.has(lowercaseKey) && !wordMap.has(line)) { if (!preferLowercase && wordMap.has(lowercaseKey) && !wordMap.has(line)) {
wordMap.set(lowercaseKey, line); wordMap.set(lowercaseKey, line);
} }
} }
@ -79,7 +89,7 @@ function printWords(wordList) {
return; return;
} }
wordList.forEach(w => console.log(w)); wordList.forEach(w => print(w));
} }

35
scripts/remove-foreign-words.js
View file

@ -3,9 +3,14 @@ const { createReadStream, existsSync } = require('fs');
const { createInterface } = require('readline'); const { createInterface } = require('readline');
function print(str) {
process.stdout.write(`${str}\n`);
}
function printHelp() { function printHelp() {
console.log(`Usage ${basename(process.argv[1])} --blacklist|whitelist DICTIONARY_LOCALE DICTIONARY.TXT FOREIGN_WORDS_LOCALE FOREIGN-WORD-DICTIONARY.txt`); print(`Usage ${basename(process.argv[1])} --blacklist|whitelist DICTIONARY_LOCALE DICTIONARY.TXT FOREIGN_WORDS_LOCALE FOREIGN-WORD-DICTIONARY.txt`);
console.log('Removes foreign words from a dictionary. "blacklist" and "whitelist" determine how the FOREIGN-WORD-DICTIONARY.txt is used.'); print('Removes foreign words from a dictionary. "blacklist" and "whitelist" determine how the FOREIGN-WORD-DICTIONARY.txt is used.');
} }
@ -42,16 +47,16 @@ function validateInput() {
async function work({ isBlacklist, locale, fileName, foreignWordsLocale, foreignWordsFileName }) { async function work({ isBlacklist, locale, fileName, foreignWordsLocale, foreignWordsFileName }) {
const foreignWords = new Set(); const originalWords = new Map();
let lineReader = createInterface({ input: createReadStream(foreignWordsFileName) }); let lineReader = createInterface({ input: createReadStream(fileName) });
for await (const line of lineReader) { for await (const line of lineReader) {
foreignWords.add(line.toLocaleLowerCase(foreignWordsLocale)); originalWords.set(line.toLocaleLowerCase(foreignWordsLocale), line);
} }
const goodWords = new Set(); const goodWords = new Set();
lineReader = createInterface({ input: createReadStream(fileName) });
lineReader = createInterface({ input: createReadStream(foreignWordsFileName) });
for await (const line of lineReader) { for await (const line of lineReader) {
if (typeof line !== 'string' || line.length === 0) { if (typeof line !== 'string' || line.length === 0) {
continue; continue;
@ -59,23 +64,23 @@ async function work({ isBlacklist, locale, fileName, foreignWordsLocale, foreign
const wordKey = line.toLocaleLowerCase(locale); const wordKey = line.toLocaleLowerCase(locale);
if ( if (isBlacklist && originalWords.has(wordKey)) {
(!isBlacklist && foreignWords.has(wordKey)) originalWords.delete(wordKey);
|| (isBlacklist && !foreignWords.has(wordKey))
) {
goodWords.add(line);
} }
if (!isBlacklist && originalWords.has(wordKey)) {
goodWords.add(line);
}
} }
return goodWords; return Array.from(isBlacklist ? originalWords.values() : goodWords);
} }
function printWords(wordList) { function printWords(wordList) {
if (wordList instanceof Set) { if (Array.isArray(wordList)) {
wordList.forEach(w => console.log(w)); wordList.forEach(w => print(w));
} }
} }