0
0
Fork 0

Fix bug 563

Bug563-HuangHaoqi
peilinwu 2024-07-04 15:37:40 +08:00
parent 00ae770195
commit 56c8dc24c6
2 changed files with 349 additions and 288 deletions

View File

@ -1,89 +1,147 @@
###########################################################################
# Copyright 2019 (C) Hui Lan <hui.lan@cantab.net>
# Written permission must be obtained from the author for commercial uses.
###########################################################################
# Purpose: dictionary & pickle as a simple means of database.
# Task: incorporate the functions into wordfreqCMD.py such that it will also show cumulative frequency.
# Note: unlike pick_idea.py, now the second item is not frequency, but a list of dates.
import pickle
from datetime import datetime
def lst2dict(lst, d):
'''
Store the information in list lst to dictionary d.
Note: nothing is returned.
'''
for x in lst:
word = x[0]
dates = x[1]
if not word in d:
d[word] = dates
else:
d[word] += dates
def deleteRecord(path,word):
with open(path, 'rb') as f:
db = pickle.load(f)
try:
db.pop(word)
except KeyError:
print("sorry")
with open(path, 'wb') as ff:
pickle.dump(db, ff)
def dict2lst(d):
if len(d) > 0:
keys = list(d.keys())
if isinstance(d[keys[0]], int):
lst = []
for k in d:
lst.append((k, [datetime.now().strftime('%Y%m%d%H%M')]))
return lst
elif isinstance(d[keys[0]], list):
return list(d.items()) # a list of (key, value) pairs
return []
def merge_frequency(lst1, lst2):
d = {}
lst2dict(lst1, d)
lst2dict(lst2, d)
return d
def load_record(pickle_fname):
f = open(pickle_fname, 'rb')
d = pickle.load(f)
f.close()
return d
def save_frequency_to_pickle(d, pickle_fname):
f = open(pickle_fname, 'wb')
exclusion_lst = ['one', 'no', 'has', 'had', 'do', 'that', 'have', 'by', 'not', 'but', 'we', 'this', 'my', 'him', 'so', 'or', 'as', 'are', 'it', 'from', 'with', 'be', 'can', 'for', 'an', 'if', 'who', 'whom', 'whose', 'which', 'the', 'to', 'a', 'of', 'and', 'you', 'i', 'he', 'she', 'they', 'me', 'was', 'were', 'is', 'in', 'at', 'on', 'their', 'his', 'her', 's', 'said', 'all', 'did', 'been', 'w']
d2 = {}
for k in d:
if not k in exclusion_lst and not k.isnumeric() and not len(k) < 2:
d2[k] = list(sorted(d[k])) # 原先这里是d2[k] = list(sorted(set(d[k])))
pickle.dump(d2, f)
f.close()
if __name__ == '__main__':
lst1 = [('apple',['201910251437', '201910251438']), ('banana',['201910251439'])]
d = {}
lst2dict(lst1, d) # d will change
save_frequency_to_pickle(d, 'frequency.p') # frequency.p is our database
lst2 = [('banana',['201910251439']), ('orange', ['201910251440', '201910251439'])]
d = load_record('frequency.p')
lst1 = dict2lst(d)
d = merge_frequency(lst2, lst1)
print(d)
###########################################################################
# Copyright 2019 (C) Hui Lan <hui.lan@cantab.net>
# Written permission must be obtained from the author for commercial uses.
###########################################################################
# Purpose: dictionary & pickle as a simple means of database.
# Task: incorporate the functions into wordfreqCMD.py such that it will also show cumulative frequency.
# Note: unlike pick_idea.py, now the second item is not frequency, but a list of dates.
import pickle
import requests
import hashlib
import random
from urllib.parse import urlencode
from datetime import datetime
class BaiduContent:
APPID = '20200314000398337' # 将'您的APPID'替换为实际的APPID
KEY = 'uZ6Sdwz_V1zu9q1peowk' # 将'您的密钥'替换为实际的API密钥
def generate_sign(appid, q, salt, key):
sign_str = appid + q + str(salt) + key
sign = hashlib.md5(sign_str.encode('utf-8')).hexdigest()
return sign
def is_valid_word(word):
url = "https://fanyi-api.baidu.com/api/trans/vip/translate"
salt = random.randint(32768, 65536)
sign = generate_sign(BaiduContent.APPID, word, salt, BaiduContent.KEY)
params = {
'q': word,
'from': 'en',
'to': 'zh',
'appid': BaiduContent.APPID,
'salt': salt,
'sign': sign
}
headers = {'Content-Type': "application/x-www-form-urlencoded"}
data = urlencode(params).encode('utf-8')
try:
response = requests.post(url, data=data, headers=headers)
if response.status_code == 200:
data = response.json()
# print(data['trans_result'][0]['dst'])
# 检查是否含有翻译结果,并判断翻译后的文本是否与原文不同
if 'trans_result' in data and data['trans_result'][0]['dst'] != word:
return True
else:
return False
else:
return False
except requests.RequestException:
return False
def lst2dict(lst, d):
'''
Store the information in list lst to dictionary d.
Note: nothing is returned.
'''
for x in lst:
word = x[0]
dates = x[1]
if not word in d:
d[word] = dates
else:
d[word] += dates
def deleteRecord(path,word):
with open(path, 'rb') as f:
db = pickle.load(f)
try:
db.pop(word)
except KeyError:
print("sorry")
with open(path, 'wb') as ff:
pickle.dump(db, ff)
def dict2lst(d):
if len(d) > 0:
keys = list(d.keys())
if isinstance(d[keys[0]], int):
lst = []
for k in d:
lst.append((k, [datetime.now().strftime('%Y%m%d%H%M')]))
return lst
elif isinstance(d[keys[0]], list):
return list(d.items()) # a list of (key, value) pairs
return []
def merge_frequency(lst1, lst2):
d = {}
lst2dict(lst1, d)
lst2dict(lst2, d)
return d
def load_record(pickle_fname):
f = open(pickle_fname, 'rb')
d = pickle.load(f)
f.close()
return d
# exclusion_lst = ['one', 'no', 'has', 'had', 'do', 'that', 'have', 'by', 'not', 'but', 'we', 'this', 'my', 'him', 'so', 'or', 'as', 'are', 'it', 'from', 'with', 'be', 'can', 'for', 'an', 'if', 'who', 'whom', 'whose', 'which', 'the', 'to', 'a', 'of', 'and', 'you', 'i', 'he', 'she', 'they', 'me', 'was', 'were', 'is', 'in', 'at', 'on', 'their', 'his', 'her', 's', 'said', 'all', 'did', 'been', 'w']
def save_frequency_to_pickle(d, pickle_fname):
with open(pickle_fname, 'wb') as f: # 使用 with 语句自动处理文件关闭
exclusion_lst = []
d2 = {}
illegal = False # 标记是否合法
added = False
for k in d:
if k not in exclusion_lst and not k.isnumeric():
if is_valid_word(k): # 只有当单词不合法时进行标记
d2[k] = list(sorted(d[k]))
added =True
else:
illegal = True # 标记至少处理了一个有效单词
if illegal:
if not added:
pickle.dump({}, f)
else:
pickle.dump(d2, f)
return 0 # 返回0表示成功处理存在非法单词
else:
pickle.dump(d2, f)
return 1 # 返回1表示成功处理并保存至少一个单词
if __name__ == '__main__':
lst1 = [('apple',['201910251437', '201910251438']), ('banana',['201910251439'])]
d = {}
lst2dict(lst1, d) # d will change
save_frequency_to_pickle(d, 'frequency.p') # frequency.p is our database
lst2 = [('banana',['201910251439']), ('orange', ['201910251440', '201910251439'])]
d = load_record('frequency.p')
lst1 = dict2lst(d)
d = merge_frequency(lst2, lst1)
print(d)

View File

@ -1,199 +1,202 @@
from datetime import datetime
from admin_service import ADMIN_NAME
from flask import *
# from app import Yaml
# from app.Article import get_today_article, load_freq_history
# from app.WordFreq import WordFreq
# from app.wordfreqCMD import sort_in_descending_order
import Yaml
from Article import get_today_article, load_freq_history
from WordFreq import WordFreq
from wordfreqCMD import sort_in_descending_order
import pickle_idea
import pickle_idea2
import logging
logging.basicConfig(filename='log.txt', format='%(asctime)s %(message)s', level=logging.DEBUG)
# 初始化蓝图
userService = Blueprint("user_bp", __name__)
path_prefix = '/var/www/wordfreq/wordfreq/'
path_prefix = './' # comment this line in deployment
@userService.route("/get_next_article/<username>",methods=['GET','POST'])
def get_next_article(username):
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username)
session['old_articleID'] = session.get('articleID')
if request.method == 'GET':
visited_articles = session.get("visited_articles")
if visited_articles['article_ids'][-1] == "null": # 如果当前还是“null”则将“null”pop出来,无需index+=1
visited_articles['article_ids'].pop()
else: # 当前不为“null”直接 index+=1
visited_articles["index"] += 1
session["visited_articles"] = visited_articles
logging.debug('/get_next_article: start calling get_today_arcile()')
visited_articles, today_article, result_of_generate_article = get_today_article(user_freq_record, session.get('visited_articles'))
logging.debug('/get_next_arcile: done.')
data = {
'visited_articles': visited_articles,
'today_article': today_article,
'result_of_generate_article': result_of_generate_article
}
else:
return 'Under construction'
return json.dumps(data)
@userService.route("/get_pre_article/<username>",methods=['GET'])
def get_pre_article(username):
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username)
if request.method == 'GET':
visited_articles = session.get("visited_articles")
if(visited_articles["index"]==0):
data=''
else:
visited_articles["index"] -= 1 # 上一篇index-=1
if visited_articles['article_ids'][-1] == "null": # 如果当前还是“null”则将“null”pop出来
visited_articles['article_ids'].pop()
session["visited_articles"] = visited_articles
visited_articles, today_article, result_of_generate_article = get_today_article(user_freq_record, session.get('visited_articles'))
data = {
'visited_articles': visited_articles,
'today_article': today_article,
'result_of_generate_article':result_of_generate_article
}
return json.dumps(data)
@userService.route("/<username>/<word>/unfamiliar", methods=['GET', 'POST'])
def unfamiliar(username, word):
'''
:param username:
:param word:
:return:
'''
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username)
pickle_idea.unfamiliar(user_freq_record, word)
session['thisWord'] = word # 1. put a word into session
session['time'] = 1
return "success"
@userService.route("/<username>/<word>/familiar", methods=['GET', 'POST'])
def familiar(username, word):
'''
:param username:
:param word:
:return:
'''
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username)
pickle_idea.familiar(user_freq_record, word)
session['thisWord'] = word # 1. put a word into session
session['time'] = 1
return "success"
@userService.route("/<username>/<word>/del", methods=['GET', 'POST'])
def deleteword(username, word):
'''
删除单词
:param username: 用户名
:param word: 单词
:return: 重定位到用户界面
'''
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username)
pickle_idea2.deleteRecord(user_freq_record, word)
# 模板userpage_get.html中删除单词是异步执行而flash的信息后续是同步执行的所以注释这段代码同时如果这里使用flash但不提取信息则会影响 signup.html的显示。bug复现删除单词后点击退出点击注册注册页面就会出现提示信息
# flash(f'{word} is no longer in your word list.')
return "success"
@userService.route("/<username>/userpage", methods=['GET', 'POST'])
def userpage(username):
'''
用户界面
:param username: 用户名
:return: 返回用户界面
'''
# 未登录,跳转到未登录界面
if not session.get('logged_in'):
return render_template('not_login.html')
# 用户过期
user_expiry_date = session.get('expiry_date')
if datetime.now().strftime('%Y%m%d') > user_expiry_date:
return render_template('expiry.html', expiry_date=user_expiry_date)
# 获取session里的用户名
username = session.get('username')
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username)
if request.method == 'POST': # when we submit a form
content = request.form['content']
f = WordFreq(content)
lst = f.get_freq()
return render_template('userpage_post.html',username=username,lst = lst, yml=Yaml.yml)
elif request.method == 'GET': # when we load a html page
d = load_freq_history(user_freq_record)
lst = pickle_idea2.dict2lst(d)
lst2 = []
for t in lst:
lst2.append((t[0], len(t[1])))
lst3 = sort_in_descending_order(lst2)
words = ''
for x in lst3:
words += x[0] + ' '
visited_articles, today_article, result_of_generate_article = get_today_article(user_freq_record, session.get('visited_articles'))
session['visited_articles'] = visited_articles
# 通过 today_article加载前端的显示页面
return render_template('userpage_get.html',
admin_name=ADMIN_NAME,
username=username,
session=session,
# flashed_messages=get_flashed_messages(), 仅有删除单词的时候使用到flash而删除单词是异步执行这里的信息提示是同步执行所以就没有存在的必要了
today_article=today_article,
result_of_generate_article=result_of_generate_article,
d_len=len(d),
lst3=lst3,
yml=Yaml.yml,
words=words)
@userService.route("/<username>/mark", methods=['GET', 'POST'])
def user_mark_word(username):
'''
标记单词
:param username: 用户名
:return: 重定位到用户界面
'''
username = session[username]
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username)
if request.method == 'POST':
# 提交标记的单词
d = load_freq_history(user_freq_record)
lst_history = pickle_idea2.dict2lst(d)
lst = []
for word in request.form.getlist('marked'):
lst.append((word, [get_time()]))
d = pickle_idea2.merge_frequency(lst, lst_history)
if len(lst_history) > 999:
flash('You have way too many words in your difficult-words book. Delete some first.')
else:
pickle_idea2.save_frequency_to_pickle(d, user_freq_record)
flash('Added %s.' % (', '.join(request.form.getlist('marked'))))
return redirect(url_for('user_bp.userpage', username=username))
else:
return 'Under construction'
def get_time():
'''
获取当前时间
:return: 当前时间
'''
return datetime.now().strftime('%Y%m%d%H%M') # upper to minutes
from datetime import datetime
from admin_service import ADMIN_NAME
from flask import *
# from app import Yaml
# from app.Article import get_today_article, load_freq_history
# from app.WordFreq import WordFreq
# from app.wordfreqCMD import sort_in_descending_order
import Yaml
from Article import get_today_article, load_freq_history
from WordFreq import WordFreq
from wordfreqCMD import sort_in_descending_order
import pickle_idea
import pickle_idea2
import logging
logging.basicConfig(filename='log.txt', format='%(asctime)s %(message)s', level=logging.DEBUG)
# 初始化蓝图
userService = Blueprint("user_bp", __name__)
path_prefix = '/var/www/wordfreq/wordfreq/'
path_prefix = './' # comment this line in deployment
@userService.route("/get_next_article/<username>",methods=['GET','POST'])
def get_next_article(username):
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username)
session['old_articleID'] = session.get('articleID')
if request.method == 'GET':
visited_articles = session.get("visited_articles")
if visited_articles['article_ids'][-1] == "null": # 如果当前还是“null”则将“null”pop出来,无需index+=1
visited_articles['article_ids'].pop()
else: # 当前不为“null”直接 index+=1
visited_articles["index"] += 1
session["visited_articles"] = visited_articles
logging.debug('/get_next_article: start calling get_today_arcile()')
visited_articles, today_article, result_of_generate_article = get_today_article(user_freq_record, session.get('visited_articles'))
logging.debug('/get_next_arcile: done.')
data = {
'visited_articles': visited_articles,
'today_article': today_article,
'result_of_generate_article': result_of_generate_article
}
else:
return 'Under construction'
return json.dumps(data)
@userService.route("/get_pre_article/<username>",methods=['GET'])
def get_pre_article(username):
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username)
if request.method == 'GET':
visited_articles = session.get("visited_articles")
if(visited_articles["index"]==0):
data=''
else:
visited_articles["index"] -= 1 # 上一篇index-=1
if visited_articles['article_ids'][-1] == "null": # 如果当前还是“null”则将“null”pop出来
visited_articles['article_ids'].pop()
session["visited_articles"] = visited_articles
visited_articles, today_article, result_of_generate_article = get_today_article(user_freq_record, session.get('visited_articles'))
data = {
'visited_articles': visited_articles,
'today_article': today_article,
'result_of_generate_article':result_of_generate_article
}
return json.dumps(data)
@userService.route("/<username>/<word>/unfamiliar", methods=['GET', 'POST'])
def unfamiliar(username, word):
'''
:param username:
:param word:
:return:
'''
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username)
pickle_idea.unfamiliar(user_freq_record, word)
session['thisWord'] = word # 1. put a word into session
session['time'] = 1
return "success"
@userService.route("/<username>/<word>/familiar", methods=['GET', 'POST'])
def familiar(username, word):
'''
:param username:
:param word:
:return:
'''
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username)
pickle_idea.familiar(user_freq_record, word)
session['thisWord'] = word # 1. put a word into session
session['time'] = 1
return "success"
@userService.route("/<username>/<word>/del", methods=['GET', 'POST'])
def deleteword(username, word):
'''
删除单词
:param username: 用户名
:param word: 单词
:return: 重定位到用户界面
'''
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username)
pickle_idea2.deleteRecord(user_freq_record, word)
# 模板userpage_get.html中删除单词是异步执行而flash的信息后续是同步执行的所以注释这段代码同时如果这里使用flash但不提取信息则会影响 signup.html的显示。bug复现删除单词后点击退出点击注册注册页面就会出现提示信息
# flash(f'{word} is no longer in your word list.')
return "success"
@userService.route("/<username>/userpage", methods=['GET', 'POST'])
def userpage(username):
'''
用户界面
:param username: 用户名
:return: 返回用户界面
'''
# 未登录,跳转到未登录界面
if not session.get('logged_in'):
return render_template('not_login.html')
# 用户过期
user_expiry_date = session.get('expiry_date')
if datetime.now().strftime('%Y%m%d') > user_expiry_date:
return render_template('expiry.html', expiry_date=user_expiry_date)
# 获取session里的用户名
username = session.get('username')
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username)
if request.method == 'POST': # when we submit a form
content = request.form['content']
f = WordFreq(content)
lst = f.get_freq()
return render_template('userpage_post.html',username=username,lst = lst, yml=Yaml.yml)
elif request.method == 'GET': # when we load a html page
d = load_freq_history(user_freq_record)
lst = pickle_idea2.dict2lst(d)
lst2 = []
for t in lst:
lst2.append((t[0], len(t[1])))
lst3 = sort_in_descending_order(lst2)
words = ''
for x in lst3:
words += x[0] + ' '
visited_articles, today_article, result_of_generate_article = get_today_article(user_freq_record, session.get('visited_articles'))
session['visited_articles'] = visited_articles
# 通过 today_article加载前端的显示页面
return render_template('userpage_get.html',
admin_name=ADMIN_NAME,
username=username,
session=session,
# flashed_messages=get_flashed_messages(), 仅有删除单词的时候使用到flash而删除单词是异步执行这里的信息提示是同步执行所以就没有存在的必要了
today_article=today_article,
result_of_generate_article=result_of_generate_article,
d_len=len(d),
lst3=lst3,
yml=Yaml.yml,
words=words)
@userService.route("/<username>/mark", methods=['GET', 'POST'])
def user_mark_word(username):
'''
标记单词
:param username: 用户名
:return: 重定位到用户界面
'''
username = session[username]
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username)
if request.method == 'POST':
# 提交标记的单词
d = load_freq_history(user_freq_record)
lst_history = pickle_idea2.dict2lst(d)
lst = []
for word in request.form.getlist('marked'):
lst.append((word, [get_time()]))
d = pickle_idea2.merge_frequency(lst, lst_history)
if len(lst_history) > 999:
flash('You have way too many words in your difficult-words book. Delete some first.')
else:
if pickle_idea2.save_frequency_to_pickle(d, user_freq_record):
flash('Added %s.' % (', '.join(request.form.getlist('marked'))))
else:
flash('%s.存在非法单词' % (', '.join(request.form.getlist('marked'))))
return redirect(url_for('user_bp.userpage', username=username))
else:
return 'Under construction'
def get_time():
'''
获取当前时间
:return: 当前时间
'''
return datetime.now().strftime('%Y%m%d%H%M') # upper to minutes