wzray/tt9
1
0
Fork 0
Code Activity

JavaScript code cleanup and optimizations

Browse source 
* removed all console statements

  * sorting script: no longer creating functions within the loop

  * aosp2tt9: fixed incorrect usage of a global variable

  * injest-words: fixed a non-strict comparison

  * remove-repeating-words: commented out the case-sensitive search function, as it may be going out of use
This commit is contained in:
sspanak 2024-03-27 17:04:09 +02:00 • committed by Dimo Karaivanov
parent 1c900139a2
commit 8a971ea15e
9 changed files with 70 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

7
scripts/_printers.js Normal file
View file

@ -0,0 +1,7 @@
exports.print = function(str) {
process.stdout.write(`${str}\n`);
};
exports.printError = function(str) {
process.stderr.write(`${str}\n`);
};

13
scripts/aosp2tt9.js
View file

@ -1,11 +1,12 @@
const { basename } = require('path');
const { createReadStream, existsSync } = require('fs');
const { createInterface } = require('readline');
const { print, printError } = require('./_printers.js');
function printHelp() {
console.log(`Usage ${basename(process.argv[1])} aosp-dictionary-file.txt [minimum-frequency] [--no-freq]`);
console.log('Converts an AOSP dictionary to TT9 compatible format. The second parameter must be an integer and allows for filtering words with frequency less than the given number. If "--no-freq" is set, only words without frequencies will be listed.');
print(`Usage ${basename(process.argv[1])} aosp-dictionary-file.txt [minimum-frequency] [--no-freq]`);
print('Converts an AOSP dictionary to TT9 compatible format. The second parameter must be an integer and allows for filtering words with frequency less than the given number. If "--no-freq" is set, only words without frequencies will be listed.');
}
@ -17,7 +18,7 @@ function validateInput() {
}
if (!existsSync(process.argv[2])) {
console.error(`Failure! Could not find dictionary file "${process.argv[3]}."`);
printError(`Failure! Could not find dictionary file "${process.argv[2]}".`);
process.exit(2);
}
@ -31,13 +32,13 @@ function validateInput() {
function printWords(wordList) {
if (Array.isArray(wordList)) {
wordList.forEach(w => console.log(w));
wordList.forEach(w => print(w));
}
}
async function convert({ fileName, minFrequency, noFrequencies }) {
words = [];
const words = [];
let lineReader = createInterface({ input: createReadStream(fileName) });
for await (const line of lineReader) {
@ -74,4 +75,4 @@ async function convert({ fileName, minFrequency, noFrequencies }) {
/** main **/
convert(validateInput())
.then(words => printWords(words))
.catch(e => console.error(e));
.catch(e => printError(e));

21
scripts/capitalize-dictionary-words.js
View file

@ -1,12 +1,13 @@
const { basename } = require('path');
const { createReadStream, existsSync } = require('fs');
const { print, printError } = require('./_printers.js');
function printHelp() {
console.log(`Usage ${basename(process.argv[1])} DICTIONARY-FILE-NAME.txt LIST-OF-CAPITALIZED-WORDS.txt MIN-WORD-LENGTH LOCALE`);
console.log('Capitalizes a word list using capitalized words in another list.');
console.log('\nMIN-WORD-LENGTH must be a positive number.');
console.log('LOCALE could be any valid JS locale, for exmaple: en, en-US, etc...');
print(`Usage ${basename(process.argv[1])} DICTIONARY-FILE-NAME.txt LIST-OF-CAPITALIZED-WORDS.txt MIN-WORD-LENGTH LOCALE`);
print('Capitalizes a word list using capitalized words in another list.');
print('\nMIN-WORD-LENGTH must be a positive number.');
print('LOCALE could be any valid JS locale, for exmaple: en, en-US, etc...');
}
@ -17,21 +18,19 @@ function validateInput() {
process.exit(1);
}
if (!existsSync(process.argv[3])) {
console.error(`Failure! Could not find list-of-capitals file "${process.argv[3]}."`);
printError(`Failure! Could not find list-of-capitals file "${process.argv[3]}".`);
process.exit(2);
}
if (!existsSync(process.argv[2])) {
console.error(`Failure! Could not find dictionary file "${process.argv[2]}."`);
printError(`Failure! Could not find dictionary file "${process.argv[2]}".`);
process.exit(2);
}
const minWordLength = Number.parseInt(process.argv[4]);
if (Number.isNaN(minWordLength) || minWordLength < 0) {
console.error(`Failure! The minimum word length must be a positive number.`);
printError(`Failure! The minimum word length must be a positive number.`);
process.exit(2);
}
@ -84,7 +83,7 @@ function printWords(wordList) {
return;
}
wordList.forEach(w => console.log(w));
wordList.forEach(w => print(w));
}
@ -92,4 +91,4 @@ function printWords(wordList) {
/** main **/
capitalize(validateInput())
.then(words => printWords(words))
.catch(e => console.error(e));
.catch(e => printError(e));

15
scripts/inject-dictionary-frequencies.js
View file

@ -1,14 +1,15 @@
const { basename } = require('path');
const { createReadStream, existsSync } = require('fs');
const { print, printError } = require('./_printers.js');
const DELIMITER = ' ';
function printHelp() {
console.log(`Usage ${basename(process.argv[1])} LOCALE DICTIONARY-FILE-NAME.txt WORDS-WITH-FREQUENCIES.txt`);
console.log('Matches up the words from DICTIONARY-FILE-NAME with the frequencies in WORDS-WITH-FREQUENCIES file.');
console.log('LOCALE could be any valid JS locale, for exmaple: en, en-US, etc...');
print(`Usage ${basename(process.argv[1])} LOCALE DICTIONARY-FILE-NAME.txt WORDS-WITH-FREQUENCIES.txt`);
print('Matches up the words from DICTIONARY-FILE-NAME with the frequencies in WORDS-WITH-FREQUENCIES file.');
print('LOCALE could be any valid JS locale, for exmaple: en, en-US, etc...');
}
@ -20,13 +21,13 @@ function validateInput() {
if (!existsSync(process.argv[4])) {
console.error(`Failure! Could not find the WORDS-WITH-FREQUENCIES file "${process.argv[4]}."`);
printError(`Failure! Could not find the WORDS-WITH-FREQUENCIES file "${process.argv[4]}".`);
process.exit(2);
}
if (!existsSync(process.argv[3])) {
console.error(`Failure! Could not find dictionary file "${process.argv[3]}."`);
printError(`Failure! Could not find dictionary file "${process.argv[3]}".`);
process.exit(2);
}
@ -79,7 +80,7 @@ async function inject({ wordsWithFrequenciesFileName, dictionaryFileName, locale
function printWords(wordList) {
if (Array.isArray(wordList)) {
wordList.forEach(w => console.log(w));
wordList.forEach(w => print(w));
}
}
@ -88,4 +89,4 @@ function printWords(wordList) {
/** main **/
inject(validateInput())
.then(words => printWords(words))
.catch(e => console.error(e));
.catch(e => printError(e));

17
scripts/injest-words.js
View file

@ -1,12 +1,13 @@
const { basename } = require('path');
const { createReadStream, existsSync } = require('fs');
const { createInterface } = require('readline');
const { print, printError } = require('./_printers.js');
function printHelp() {
console.log(`Usage ${basename(process.argv[1])} word-list.txt [split-ignore-list.txt]`);
console.log('Breaks dashed words into separate words, puts multiple words on a line on new lines and deletes repeating new lines.');
console.log('The split-ignore-list is optional. Allows for not splitting certain words by dashes.');
print(`Usage ${basename(process.argv[1])} word-list.txt [split-ignore-list.txt]`);
print('Breaks dashed words into separate words, puts multiple words on a line on new lines and deletes repeating new lines.');
print('The split-ignore-list is optional. Allows for not splitting certain words by dashes.');
}
@ -18,12 +19,12 @@ function validateInput() {
}
if (!existsSync(process.argv[2])) {
console.error(`Failure! Could not find word list file "${process.argv[3]}."`);
printError(`Failure! Could not find word list file "${process.argv[2]}".`);
process.exit(2);
}
if (process.argv[3] && !existsSync(process.argv[3])) {
console.error(`Failure! Could not ignore list file "${process.argv[3]}."`);
printError(`Failure! Could not find ignore list file "${process.argv[3]}".`);
process.exit(2);
}
@ -36,7 +37,7 @@ function validateInput() {
function printWords(wordList) {
if (Array.isArray(wordList)) {
wordList.forEach(w => console.log(w));
wordList.forEach(w => print(w));
}
}
@ -72,7 +73,7 @@ function splitDashedWords(inputWords, ignoreList) {
}
const [root, ...others] = word.split('-');
if (root === undefined || others.length != 1) {
if (root === undefined || others.length !== 1) {
continue;
}
@ -125,4 +126,4 @@ async function work({ fileName, ignoreListFileName }) {
/** main **/
work(validateInput())
.then(words => printWords(words))
.catch(e => console.error(e));
.catch(e => printError(e));

16
scripts/remove-dictionary-repeating-words.js
View file

@ -1,13 +1,7 @@
const { basename } = require('path');
const { createReadStream, existsSync } = require('fs');
const { createInterface } = require('readline');
const GEO_NAME = /[A-Z]\w+\-[^\n]+/;
function print(str) {
process.stdout.write(`${str}\n`);
}
const { print, printError } = require('./_printers.js');
function printHelp() {
@ -27,7 +21,7 @@ function validateInput() {
if (!existsSync(process.argv[3])) {
console.error(`Failure! Could not find file "${process.argv[3]}."`);
printError(`Failure! Could not find file "${process.argv[3]}".`);
process.exit(2);
}
@ -39,6 +33,8 @@ function validateInput() {
}
/*
const GEO_NAME = /[A-Z]\w+-[^\n]+/;
function getRegularWordKey(locale, word) {
if (typeof word !== 'string' || word.length === 0) {
@ -47,7 +43,7 @@ function getRegularWordKey(locale, word) {
return GEO_NAME.test(word) ? word : word.toLocaleLowerCase(locale);
}
*/
function getLowercaseWordKey(locale, word) {
@ -97,4 +93,4 @@ function printWords(wordList) {
/** main **/
removeRepeatingWords(validateInput())
.then(words => printWords(words))
.catch(e => console.error(e));
.catch(e => printError(e));

15
scripts/remove-extra-dashes.js
View file

@ -1,13 +1,14 @@
const { basename } = require('path');
const { createReadStream, existsSync } = require('fs');
const { createInterface } = require('readline');
const { print, printError } = require('./_printers.js');
function printHelp() {
console.log(`Usage ${basename(process.argv[1])} LOCALE word-list.txt`);
console.log('Searches for compound words with that also exsit as separate words and removes the compound variants.');
console.log('For example, "fly-by" will be removed, if the word list contains both "fly" and "by".')
console.log('LOCALE could be any valid JS locale, for exmaple: en, en-US, etc...')
print(`Usage ${basename(process.argv[1])} LOCALE word-list.txt`);
print('Searches for compound words with that also exsit as separate words and removes the compound variants.');
print('For example, "fly-by" will be removed, if the word list contains both "fly" and "by".')
print('LOCALE could be any valid JS locale, for exmaple: en, en-US, etc...')
}
@ -18,7 +19,7 @@ function validateInput() {
}
if (!existsSync(process.argv[3])) {
console.error(`Failure! Could not find word list file "${process.argv[3]}."`);
printError(`Failure! Could not find word list file "${process.argv[3]}".`);
process.exit(2);
}
@ -32,7 +33,7 @@ function validateInput() {
function printWords(wordList) {
if (wordList instanceof Set) {
wordList.forEach(w => console.log(w));
wordList.forEach(w => print(w));
}
}
@ -109,4 +110,4 @@ async function work({ fileName, locale, separator }) {
/** main **/
work(validateInput())
.then(words => printWords(words))
.catch(e => console.error(e));
.catch(e => printError(e));

14
scripts/remove-foreign-words.js
View file

@ -1,11 +1,7 @@
const { basename } = require('path');
const { createReadStream, existsSync } = require('fs');
const { createInterface } = require('readline');
function print(str) {
process.stdout.write(`${str}\n`);
}
const { print, printError } = require('./_printers.js');
function printHelp() {
@ -22,17 +18,17 @@ function validateInput() {
}
if (process.argv[2] !== '--blacklist' && process.argv[2] !== '--whitelist') {
console.error(`Failure! You must specify whether to use the foreign words file as a blacklist or a whitelist."`);
printError(`Failure! You must specify whether to use the foreign words file as a blacklist or a whitelist."`);
process.exit(3);
}
if (!existsSync(process.argv[4])) {
console.error(`Failure! Could not find words file "${process.argv[4]}."`);
printError(`Failure! Could not find words file "${process.argv[4]}".`);
process.exit(2);
}
if (!existsSync(process.argv[6])) {
console.error(`Failure! Could not find foreign words file "${process.argv[6]}."`);
printError(`Failure! Could not find foreign words file "${process.argv[6]}".`);
process.exit(2);
}
@ -89,4 +85,4 @@ function printWords(wordList) {
/** main **/
work(validateInput())
.then(words => printWords(words))
.catch(e => console.error(e));
.catch(e => printError(e));

17
scripts/sort-dictionary.js
View file

@ -1,11 +1,12 @@
const { basename } = require('path');
const { createReadStream, existsSync } = require('fs');
const { createInterface } = require('readline');
const { print, printError } = require('./_printers.js');
function printHelp() {
console.log(`Usage ${basename(process.argv[1])} LOCALE WORD-LIST.txt LANGUAGE-DEFINITION.yml`);
console.log('Sorts a dictionary for optimum search speed.');
print(`Usage ${basename(process.argv[1])} LOCALE WORD-LIST.txt LANGUAGE-DEFINITION.yml`);
print('Sorts a dictionary for optimum search speed.');
}
@ -17,12 +18,12 @@ function validateInput() {
}
if (!existsSync(process.argv[3])) {
console.error(`Failure! Could not find word list file "${process.argv[3]}."`);
printError(`Failure! Could not find word list file "${process.argv[3]}".`);
process.exit(2);
}
if (!existsSync(process.argv[4])) {
console.error(`Failure! Could not find language definition file "${process.argv[3]}."`);
printError(`Failure! Could not find language definition file "${process.argv[4]}".`);
process.exit(2);
}
@ -36,7 +37,7 @@ function validateInput() {
function printWords(wordList) {
if (Array.isArray(wordList)) {
wordList.forEach(w => console.log(`${w.word}${w.frequency ? '\t' + w.frequency : ''}`));
wordList.forEach(w => print(`${w.word}${w.frequency ? '\t' + w.frequency : ''}`));
}
}
@ -77,7 +78,9 @@ async function readDefinition(fileName) {
const matches = line.match(lettersPattern);
if (matches && matches[1]) {
const letters = matches[1].replace(/\s/g, '').split(',');
letters.forEach(l => letterWeights.set(l, key));
for (let l of letters) {
letterWeights.set(l, key);
}
key++;
}
}
@ -120,4 +123,4 @@ async function work({ definitionFile, wordsFile, locale }) {
/** main **/
work(validateInput())
.then(words => printWords(words))
.catch(e => console.error(e));
.catch(e => printError(e));