From 4a5fc9a7ce6c6d5c6fde6ffea813be9db70e96b5 Mon Sep 17 00:00:00 2001
From: AsWhitale <1838528479@qq.com>
Date: Fri, 6 Jun 2025 21:00:29 +0800
Subject: [PATCH] =?UTF-8?q?code=20review=201.=E8=B0=83=E6=95=B4=E4=BA=86?=
=?UTF-8?q?=E4=B8=80=E4=BA=9B=E6=96=B9=E6=B3=95=E5=92=8C=E7=B1=BB=E7=9A=84?=
=?UTF-8?q?=E5=85=B3=E7=B3=BB=202.=E4=BC=98=E5=8C=96=E4=BA=86=E6=95=B0?=
=?UTF-8?q?=E6=8D=AE=E5=8A=A0=E8=BD=BD=203.=E4=BF=AE=E6=AD=A3=E4=BA=86?=
=?UTF-8?q?=E9=94=99=E8=AF=AF=E7=9A=84=E6=96=87=E4=BB=B6=E8=B7=AF=E5=BE=84?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
app/vocabulary.py | 195 +++++++++++++++++++++++++---------------------
1 file changed, 106 insertions(+), 89 deletions(-)
diff --git a/app/vocabulary.py b/app/vocabulary.py
index c8f3cf2..029d905 100644
--- a/app/vocabulary.py
+++ b/app/vocabulary.py
@@ -1,76 +1,13 @@
-"""
-词汇难度评估系统
-功能：根据单词在不同考试中的出现情况评估其难度级别，并计算用户或文章的词汇水平
-"""
-
import re
import pickle
+import os
from typing import Dict, List, Tuple, Union
+from collections import Counter
# 预编译正则表达式提高性能
WORD_PATTERN = re.compile(r'\b[\w-]+\b')
-def load_record(pickle_fname: str) -> Dict[str, List[str]]:
- """
- 加载pickle格式的单词-考试类型数据
-
- 参数:
- pickle_fname: pickle文件名
-
- 返回:
- 字典格式的单词到考试类型列表的映射
-
- 异常:
- FileNotFoundError: 当文件不存在时抛出
- ValueError: 当pickle文件损坏时抛出
- """
- try:
- with open(pickle_fname, 'rb') as f:
- return pickle.load(f)
- except FileNotFoundError:
- raise FileNotFoundError(f"Pickle文件 {pickle_fname} 未找到")
- except pickle.PickleError:
- raise ValueError(f"Pickle文件 {pickle_fname} 损坏或格式不正确")
-
-
-def convert_test_type_to_difficulty_level(d: Dict[str, List[str]]) -> Dict[str, int]:
- """
- 将考试类型映射为难度级别
-
- 难度级别定义:
- 0: 未知/未分类
- 4: CET4
- 5: OXFORD3000
- 6: CET6或GRADUATE
- 7: IELTS或OXFORD5000
- 8: BBC
-
- 参数:
- d: 单词到考试类型列表的映射
-
- 返回:
- 单词到难度级别的映射
- """
- result = {}
- for word, test_types in d.items():
- if 'CET4' in test_types:
- result[word] = 4
- elif 'OXFORD3000' in test_types:
- result[word] = 5
- elif 'CET6' in test_types or 'GRADUATE' in test_types:
- result[word] = 6
- elif 'IELTS' in test_types:
- result[word] = 7
- elif 'OXFORD5000' in test_types:
- result[word] = 7
- elif 'BBC' in test_types:
- result[word] = 8
- else:
- result[word] = 0
- return result
-
-
class VocabularyLevelEstimator:
"""
词汇难度评估基类
@@ -82,13 +19,77 @@ class VocabularyLevelEstimator:
"""
_test_raw = None
_difficulty_dict = None
+ PICKLE_PATH = 'static/words_and_tests.p' # 默认数据文件路径
@classmethod
def _load_data(cls):
"""延迟加载数据，避免不必要的文件操作"""
if cls._test_raw is None:
- cls._test_raw = load_record('words_and_tests.p')
- cls._difficulty_dict = convert_test_type_to_difficulty_level(cls._test_raw)
+ cls._test_raw = cls.load_record(cls.PICKLE_PATH)
+ cls._difficulty_dict = cls.convert_test_type_to_difficulty_level(cls._test_raw)
+
+ @staticmethod
+ def load_record(pickle_fname: str) -> Dict[str, List[str]]:
+ """
+ 加载pickle格式的单词-考试类型数据
+
+ 参数:
+ pickle_fname: pickle文件名
+
+ 返回:
+ 字典格式的单词到考试类型列表的映射
+
+ 异常:
+ FileNotFoundError: 当文件不存在时抛出
+ ValueError: 当pickle文件损坏时抛出
+ """
+ try:
+ # 文件校验
+ if not os.path.exists(pickle_fname):
+ raise FileNotFoundError(f"词汇数据文件 {pickle_fname} 未找到")
+ if not pickle_fname.endswith('.p'):
+ raise ValueError("仅支持.pickle格式文件")
+
+ with open(pickle_fname, 'rb') as f:
+ return pickle.load(f)
+ except pickle.PickleError as e:
+ raise ValueError(f"Pickle文件 {pickle_fname} 损坏: {str(e)}")
+
+ @staticmethod
+ def convert_test_type_to_difficulty_level(d: Dict[str, List[str]]) -> Dict[str, int]:
+ """
+ 将考试类型映射为难度级别
+
+ 难度级别定义:
+ 0: 未知/未分类
+ 4: CET4
+ 5: OXFORD3000
+ 6: CET6或GRADUATE
+ 7: IELTS或OXFORD5000
+ 8: BBC
+
+ 参数:
+ d: 单词到考试类型列表的映射
+
+ 返回:
+ 单词到难度级别的映射
+ """
+ result = {}
+ for word, test_types in d.items():
+ word_lower = word.lower() # 统一小写处理
+ if 'CET4' in test_types:
+ result[word_lower] = 4
+ elif 'OXFORD3000' in test_types:
+ result[word_lower] = 5
+ elif 'CET6' in test_types or 'GRADUATE' in test_types:
+ result[word_lower] = 6
+ elif 'IELTS' in test_types or 'OXFORD5000' in test_types:
+ result[word_lower] = 7
+ elif 'BBC' in test_types:
+ result[word_lower] = 8
+ else:
+ result[word_lower] = 0
+ return result
@classmethod
def get_word_level(cls, word: str) -> int:
@@ -102,27 +103,32 @@ class VocabularyLevelEstimator:
单词的难度级别(0-8)
"""
cls._load_data()
- return cls._difficulty_dict.get(word, 0)
+ return cls._difficulty_dict.get(word.lower(), 0)
+
+ @classmethod
+ def reload_data(cls, new_path=None):
+ """强制重新加载词汇数据"""
+ if new_path:
+ cls.PICKLE_PATH = new_path
+ cls._test_raw = None
+ cls._difficulty_dict = None
+ cls._load_data()
class UserVocabularyLevel(VocabularyLevelEstimator):
- """
- 用户词汇水平评估
- 根据用户最近查询的单词评估其词汇水平
- """
+ """用户词汇水平评估"""
- def __init__(self, d: Dict[str, List[int]]):
+ def __init__(self, user_data: Dict[str, List[int]]):
"""
初始化用户词汇数据
参数:
- d: 单词到时间戳列表的映射
+ user_data: 单词到时间戳列表的映射
"""
- self.d = d
# 获取每个单词的最新查询时间并排序
- word_time = [(word, max(times)) for word, times in d.items() if times]
+ word_time = [(word, max(times)) for word, times in user_data.items() if times]
sorted_words = sorted(word_time, key=lambda x: x[1], reverse=True)
- self.recent_words = [word for word, _ in sorted_words[:3]]
+ self.recent_words = [word for word, _ in sorted_words[:3]] # 取最近3个单词
@property
def level(self) -> float:
@@ -139,10 +145,7 @@ class UserVocabularyLevel(VocabularyLevelEstimator):
class ArticleVocabularyLevel(VocabularyLevelEstimator):
- """
- 文章词汇水平评估
- 根据文章中出现的最高难度单词评估文章词汇水平
- """
+ """文章词汇水平评估"""
def __init__(self, content: str):
"""
@@ -150,19 +153,19 @@ class ArticleVocabularyLevel(VocabularyLevelEstimator):
参数:
content: 文章内容字符串
-
- 异常:
- ValueError: 当内容为空或不是字符串时抛出
"""
if not content or not isinstance(content, str):
- raise ValueError("文章内容必须是非空字符串")
+ self.top_levels = []
+ return
- self.content = content
- # 提取所有单词并计算难度
+ # 文本预处理：转换为小写并提取单词
words = WORD_PATTERN.findall(content.lower())
+
+ # 计算单词难度并筛选有效值
word_levels = [self.get_word_level(word) for word in words]
- # 筛选有效难度并排序
valid_levels = sorted([lvl for lvl in word_levels if lvl > 0], reverse=True)
+
+ # 取难度最高的5个单词
self.top_levels = valid_levels[:5] if valid_levels else []
@property
@@ -176,4 +179,18 @@ class ArticleVocabularyLevel(VocabularyLevelEstimator):
"""
if not self.top_levels:
return 0
- return sum(self.top_levels) / len(self.top_levels)
\ No newline at end of file
+ return sum(self.top_levels) / len(self.top_levels)
+
+ def word_frequency(self, top_n=10) -> Dict[str, int]:
+ """
+ 获取文章词频统计
+
+ 参数:
+ top_n: 返回的最高频单词数量
+
+ 返回:
+ 词频最高的top_n个单词及其频率
+ """
+ words = WORD_PATTERN.findall(self.content.lower())
+ word_freq = Counter(words)
+ return dict(word_freq.most_common(top_n))