forked from mrlan/EnglishPal
123 lines
3.4 KiB
Python
123 lines
3.4 KiB
Python
'''
|
||
Estimate a user's vocabulary level given his vocabulary data
|
||
Estimate an English article's difficulty level given its content
|
||
Preliminary design
|
||
|
||
Hui, 2024-09-23
|
||
Last upated: 2024-09-25, 2024-09-30
|
||
'''
|
||
|
||
import pickle
|
||
import re
|
||
from collections import defaultdict
|
||
|
||
|
||
def load_record(pickle_fname):
|
||
with open(pickle_fname, 'rb') as f:
|
||
d = pickle.load(f)
|
||
return d
|
||
|
||
|
||
class VocabularyLevelEstimator:
|
||
_test = load_record('words_and_tests.p') # 单词到来源的映射
|
||
_source_levels = { # 来源到难度分数的映射
|
||
'BBC': 1,
|
||
'CET4': 2,
|
||
'CET6': 3,
|
||
'GRADUATE': 4,
|
||
'OXFORD3000': 1,
|
||
'TOEFL': 5,
|
||
'IELTS': 5,
|
||
'GRE': 7
|
||
}
|
||
|
||
def get_word_level(self, word):
|
||
"""获取单词难度分数"""
|
||
if word in self._test:
|
||
sources = self._test[word]
|
||
word_levels = [
|
||
self._source_levels[src]
|
||
for src in sources
|
||
if src in self._source_levels
|
||
]
|
||
if word_levels:
|
||
# 使用最高分
|
||
return max(word_levels)
|
||
return 0 # 未知单词难度为0
|
||
|
||
|
||
class UserVocabularyLevel(VocabularyLevelEstimator):
|
||
def __init__(self, d, recent_count=3):
|
||
self.d = d
|
||
# 按时间戳排序（最新的在前）
|
||
sorted_words = sorted(d.items(), key=lambda x: max(x[1]), reverse=True)
|
||
# 取最近的单词（默认3个）
|
||
self.word_lst = [word for word, _ in sorted_words[:recent_count]]
|
||
|
||
@property
|
||
def level(self):
|
||
if not self.word_lst:
|
||
return 0.0
|
||
|
||
# 使用最高分
|
||
max_score = 0
|
||
for word in self.word_lst:
|
||
score = self.get_word_level(word)
|
||
if score > max_score:
|
||
max_score = score
|
||
return max_score
|
||
|
||
|
||
class ArticleVocabularyLevel(VocabularyLevelEstimator):
|
||
def __init__(self, content):
|
||
self.content = content
|
||
# 更智能的分词，处理连字符和缩写
|
||
words = re.findall(r'\b[\w-]+\b', content.lower())
|
||
|
||
# 计算每个单词的频率和分数
|
||
word_freq = defaultdict(int)
|
||
word_scores = {}
|
||
|
||
for word in words:
|
||
if word.isalpha():
|
||
word_freq[word] += 1
|
||
if word not in word_scores:
|
||
word_scores[word] = self.get_word_level(word)
|
||
|
||
# 计算加权分数（频率 * 分数）
|
||
weighted_scores = []
|
||
for word, score in word_scores.items():
|
||
if score > 0:
|
||
weighted_scores.append((score * word_freq[word], score, word))
|
||
|
||
# 如果没有有效单词，直接返回
|
||
if not weighted_scores:
|
||
self.difficult_words = []
|
||
return
|
||
|
||
# 按加权分数排序
|
||
weighted_scores.sort(reverse=True)
|
||
|
||
# 只保留前20%的单词（至少5个，最多15个）
|
||
num_top_words = max(5, min(15, len(weighted_scores) // 5))
|
||
self.difficult_words = [score for _, score, _ in weighted_scores[:num_top_words]]
|
||
|
||
@property
|
||
def level(self):
|
||
if not self.difficult_words:
|
||
return 0.0
|
||
|
||
# 使用最高分
|
||
return max(self.difficult_words)
|
||
|
||
if __name__ == '__main__':
|
||
d = load_record('frequency_mrlan85.pickle')
|
||
print(d)
|
||
user = UserVocabularyLevel(d)
|
||
print(user.level) # level is a property
|
||
article = ArticleVocabularyLevel('This is an interesting article')
|
||
print(article.level)
|
||
|
||
|
||
|