wzray/tt9
1
0
Fork 0
Code Activity

the word normalization script now uses logarithmic scale instead of linear

Browse source
This commit is contained in:
sspanak 2024-04-29 15:09:01 +03:00 • committed by Dimo Karaivanov
parent d500980042
commit e6e5908dbc
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
scripts/normalize-frequencies.js
View file

@ -47,10 +47,10 @@ async function normalize({ fileName, maxAllowedFrequency }) {
words.push({word, frequency});
}
const normalizationRatio = maxAllowedFrequency / maxWordFrequency;
const normalizationRatio = maxAllowedFrequency / Math.log(maxWordFrequency);
for (word of words) {
word.frequency = Math.ceil(word.frequency * normalizationRatio);
word.frequency = Math.ceil(Math.log(word.frequency) * normalizationRatio);
}
return words;