diff --git a/app/pickle_idea2.py b/app/pickle_idea2.py index 0da55bc..cbee5df 100644 --- a/app/pickle_idea2.py +++ b/app/pickle_idea2.py @@ -1,89 +1,147 @@ -########################################################################### -# Copyright 2019 (C) Hui Lan -# Written permission must be obtained from the author for commercial uses. -########################################################################### - - -# Purpose: dictionary & pickle as a simple means of database. -# Task: incorporate the functions into wordfreqCMD.py such that it will also show cumulative frequency. -# Note: unlike pick_idea.py, now the second item is not frequency, but a list of dates. - -import pickle -from datetime import datetime - -def lst2dict(lst, d): - ''' - Store the information in list lst to dictionary d. - Note: nothing is returned. - - ''' - for x in lst: - word = x[0] - dates = x[1] - if not word in d: - d[word] = dates - else: - d[word] += dates - -def deleteRecord(path,word): - with open(path, 'rb') as f: - db = pickle.load(f) - try: - db.pop(word) - except KeyError: - print("sorry") - with open(path, 'wb') as ff: - pickle.dump(db, ff) - -def dict2lst(d): - if len(d) > 0: - keys = list(d.keys()) - if isinstance(d[keys[0]], int): - lst = [] - for k in d: - lst.append((k, [datetime.now().strftime('%Y%m%d%H%M')])) - return lst - elif isinstance(d[keys[0]], list): - return list(d.items()) # a list of (key, value) pairs - - return [] - -def merge_frequency(lst1, lst2): - d = {} - lst2dict(lst1, d) - lst2dict(lst2, d) - return d - - -def load_record(pickle_fname): - f = open(pickle_fname, 'rb') - d = pickle.load(f) - f.close() - return d - - -def save_frequency_to_pickle(d, pickle_fname): - f = open(pickle_fname, 'wb') - exclusion_lst = ['one', 'no', 'has', 'had', 'do', 'that', 'have', 'by', 'not', 'but', 'we', 'this', 'my', 'him', 'so', 'or', 'as', 'are', 'it', 'from', 'with', 'be', 'can', 'for', 'an', 'if', 'who', 'whom', 'whose', 'which', 'the', 'to', 'a', 'of', 'and', 'you', 'i', 'he', 'she', 'they', 'me', 'was', 'were', 'is', 'in', 'at', 'on', 'their', 'his', 'her', 's', 'said', 'all', 'did', 'been', 'w'] - d2 = {} - for k in d: - if not k in exclusion_lst and not k.isnumeric() and not len(k) < 2: - d2[k] = list(sorted(d[k])) # 原先这里是d2[k] = list(sorted(set(d[k]))) - pickle.dump(d2, f) - f.close() - - - -if __name__ == '__main__': - - lst1 = [('apple',['201910251437', '201910251438']), ('banana',['201910251439'])] - d = {} - lst2dict(lst1, d) # d will change - save_frequency_to_pickle(d, 'frequency.p') # frequency.p is our database - - - lst2 = [('banana',['201910251439']), ('orange', ['201910251440', '201910251439'])] - d = load_record('frequency.p') - lst1 = dict2lst(d) - d = merge_frequency(lst2, lst1) - print(d) +########################################################################### +# Copyright 2019 (C) Hui Lan +# Written permission must be obtained from the author for commercial uses. +########################################################################### + + +# Purpose: dictionary & pickle as a simple means of database. +# Task: incorporate the functions into wordfreqCMD.py such that it will also show cumulative frequency. +# Note: unlike pick_idea.py, now the second item is not frequency, but a list of dates. + +import pickle +import requests +import hashlib +import random +from urllib.parse import urlencode + +from datetime import datetime + +class BaiduContent: + APPID = '20200314000398337' # 将'您的APPID'替换为实际的APPID + KEY = 'uZ6Sdwz_V1zu9q1peowk' # 将'您的密钥'替换为实际的API密钥 + +def generate_sign(appid, q, salt, key): + sign_str = appid + q + str(salt) + key + sign = hashlib.md5(sign_str.encode('utf-8')).hexdigest() + return sign +def is_valid_word(word): + url = "https://fanyi-api.baidu.com/api/trans/vip/translate" + salt = random.randint(32768, 65536) + sign = generate_sign(BaiduContent.APPID, word, salt, BaiduContent.KEY) + + params = { + 'q': word, + 'from': 'en', + 'to': 'zh', + 'appid': BaiduContent.APPID, + 'salt': salt, + 'sign': sign + } + + headers = {'Content-Type': "application/x-www-form-urlencoded"} + data = urlencode(params).encode('utf-8') + + try: + response = requests.post(url, data=data, headers=headers) + if response.status_code == 200: + data = response.json() + # print(data['trans_result'][0]['dst']) + # 检查是否含有翻译结果,并判断翻译后的文本是否与原文不同 + if 'trans_result' in data and data['trans_result'][0]['dst'] != word: + return True + else: + return False + else: + return False + except requests.RequestException: + return False + +def lst2dict(lst, d): + ''' + Store the information in list lst to dictionary d. + Note: nothing is returned. + + ''' + for x in lst: + word = x[0] + dates = x[1] + if not word in d: + d[word] = dates + else: + d[word] += dates + +def deleteRecord(path,word): + with open(path, 'rb') as f: + db = pickle.load(f) + try: + db.pop(word) + except KeyError: + print("sorry") + with open(path, 'wb') as ff: + pickle.dump(db, ff) + +def dict2lst(d): + if len(d) > 0: + keys = list(d.keys()) + if isinstance(d[keys[0]], int): + lst = [] + for k in d: + lst.append((k, [datetime.now().strftime('%Y%m%d%H%M')])) + return lst + elif isinstance(d[keys[0]], list): + return list(d.items()) # a list of (key, value) pairs + + return [] + +def merge_frequency(lst1, lst2): + d = {} + lst2dict(lst1, d) + lst2dict(lst2, d) + return d + + +def load_record(pickle_fname): + f = open(pickle_fname, 'rb') + d = pickle.load(f) + f.close() + return d + + # exclusion_lst = ['one', 'no', 'has', 'had', 'do', 'that', 'have', 'by', 'not', 'but', 'we', 'this', 'my', 'him', 'so', 'or', 'as', 'are', 'it', 'from', 'with', 'be', 'can', 'for', 'an', 'if', 'who', 'whom', 'whose', 'which', 'the', 'to', 'a', 'of', 'and', 'you', 'i', 'he', 'she', 'they', 'me', 'was', 'were', 'is', 'in', 'at', 'on', 'their', 'his', 'her', 's', 'said', 'all', 'did', 'been', 'w'] +def save_frequency_to_pickle(d, pickle_fname): + with open(pickle_fname, 'wb') as f: # 使用 with 语句自动处理文件关闭 + exclusion_lst = [] + d2 = {} + illegal = False # 标记是否合法 + added = False + for k in d: + if k not in exclusion_lst and not k.isnumeric(): + if is_valid_word(k): # 只有当单词不合法时进行标记 + d2[k] = list(sorted(d[k])) + added =True + else: + illegal = True # 标记至少处理了一个有效单词 + + if illegal: + if not added: + pickle.dump({}, f) + else: + pickle.dump(d2, f) + return 0 # 返回0表示成功处理存在非法单词 + else: + pickle.dump(d2, f) + return 1 # 返回1表示成功处理并保存至少一个单词 + + +if __name__ == '__main__': + + lst1 = [('apple',['201910251437', '201910251438']), ('banana',['201910251439'])] + d = {} + lst2dict(lst1, d) # d will change + save_frequency_to_pickle(d, 'frequency.p') # frequency.p is our database + + + lst2 = [('banana',['201910251439']), ('orange', ['201910251440', '201910251439'])] + d = load_record('frequency.p') + lst1 = dict2lst(d) + d = merge_frequency(lst2, lst1) + print(d) diff --git a/app/user_service.py b/app/user_service.py index 27323b8..55fa9a5 100644 --- a/app/user_service.py +++ b/app/user_service.py @@ -1,199 +1,202 @@ -from datetime import datetime -from admin_service import ADMIN_NAME -from flask import * - -# from app import Yaml -# from app.Article import get_today_article, load_freq_history -# from app.WordFreq import WordFreq -# from app.wordfreqCMD import sort_in_descending_order - -import Yaml -from Article import get_today_article, load_freq_history -from WordFreq import WordFreq -from wordfreqCMD import sort_in_descending_order - -import pickle_idea -import pickle_idea2 - -import logging -logging.basicConfig(filename='log.txt', format='%(asctime)s %(message)s', level=logging.DEBUG) - -# 初始化蓝图 -userService = Blueprint("user_bp", __name__) - -path_prefix = '/var/www/wordfreq/wordfreq/' -path_prefix = './' # comment this line in deployment - -@userService.route("/get_next_article/",methods=['GET','POST']) -def get_next_article(username): - user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) - session['old_articleID'] = session.get('articleID') - if request.method == 'GET': - visited_articles = session.get("visited_articles") - if visited_articles['article_ids'][-1] == "null": # 如果当前还是“null”,则将“null”pop出来,无需index+=1 - visited_articles['article_ids'].pop() - else: # 当前不为“null”,直接 index+=1 - visited_articles["index"] += 1 - session["visited_articles"] = visited_articles - logging.debug('/get_next_article: start calling get_today_arcile()') - visited_articles, today_article, result_of_generate_article = get_today_article(user_freq_record, session.get('visited_articles')) - logging.debug('/get_next_arcile: done.') - data = { - 'visited_articles': visited_articles, - 'today_article': today_article, - 'result_of_generate_article': result_of_generate_article - } - else: - return 'Under construction' - return json.dumps(data) - -@userService.route("/get_pre_article/",methods=['GET']) -def get_pre_article(username): - user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) - if request.method == 'GET': - visited_articles = session.get("visited_articles") - if(visited_articles["index"]==0): - data='' - else: - visited_articles["index"] -= 1 # 上一篇,index-=1 - if visited_articles['article_ids'][-1] == "null": # 如果当前还是“null”,则将“null”pop出来 - visited_articles['article_ids'].pop() - session["visited_articles"] = visited_articles - visited_articles, today_article, result_of_generate_article = get_today_article(user_freq_record, session.get('visited_articles')) - data = { - 'visited_articles': visited_articles, - 'today_article': today_article, - 'result_of_generate_article':result_of_generate_article - } - return json.dumps(data) - -@userService.route("///unfamiliar", methods=['GET', 'POST']) -def unfamiliar(username, word): - ''' - - :param username: - :param word: - :return: - ''' - user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) - pickle_idea.unfamiliar(user_freq_record, word) - session['thisWord'] = word # 1. put a word into session - session['time'] = 1 - return "success" - - -@userService.route("///familiar", methods=['GET', 'POST']) -def familiar(username, word): - ''' - - :param username: - :param word: - :return: - ''' - user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) - pickle_idea.familiar(user_freq_record, word) - session['thisWord'] = word # 1. put a word into session - session['time'] = 1 - return "success" - - -@userService.route("///del", methods=['GET', 'POST']) -def deleteword(username, word): - ''' - 删除单词 - :param username: 用户名 - :param word: 单词 - :return: 重定位到用户界面 - ''' - user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) - pickle_idea2.deleteRecord(user_freq_record, word) - # 模板userpage_get.html中删除单词是异步执行,而flash的信息后续是同步执行的,所以注释这段代码;同时如果这里使用flash但不提取信息,则会影响 signup.html的显示。bug复现:删除单词后,点击退出,点击注册,注册页面就会出现提示信息 - # flash(f'{word} is no longer in your word list.') - return "success" - - -@userService.route("//userpage", methods=['GET', 'POST']) -def userpage(username): - ''' - 用户界面 - :param username: 用户名 - :return: 返回用户界面 - ''' - # 未登录,跳转到未登录界面 - if not session.get('logged_in'): - return render_template('not_login.html') - - # 用户过期 - user_expiry_date = session.get('expiry_date') - if datetime.now().strftime('%Y%m%d') > user_expiry_date: - return render_template('expiry.html', expiry_date=user_expiry_date) - - # 获取session里的用户名 - username = session.get('username') - - user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) - - if request.method == 'POST': # when we submit a form - content = request.form['content'] - f = WordFreq(content) - lst = f.get_freq() - return render_template('userpage_post.html',username=username,lst = lst, yml=Yaml.yml) - - elif request.method == 'GET': # when we load a html page - d = load_freq_history(user_freq_record) - lst = pickle_idea2.dict2lst(d) - lst2 = [] - for t in lst: - lst2.append((t[0], len(t[1]))) - lst3 = sort_in_descending_order(lst2) - words = '' - for x in lst3: - words += x[0] + ' ' - visited_articles, today_article, result_of_generate_article = get_today_article(user_freq_record, session.get('visited_articles')) - session['visited_articles'] = visited_articles - # 通过 today_article,加载前端的显示页面 - return render_template('userpage_get.html', - admin_name=ADMIN_NAME, - username=username, - session=session, - # flashed_messages=get_flashed_messages(), 仅有删除单词的时候使用到flash,而删除单词是异步执行,这里的信息提示是同步执行,所以就没有存在的必要了 - today_article=today_article, - result_of_generate_article=result_of_generate_article, - d_len=len(d), - lst3=lst3, - yml=Yaml.yml, - words=words) - -@userService.route("//mark", methods=['GET', 'POST']) -def user_mark_word(username): - ''' - 标记单词 - :param username: 用户名 - :return: 重定位到用户界面 - ''' - username = session[username] - user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) - if request.method == 'POST': - # 提交标记的单词 - d = load_freq_history(user_freq_record) - lst_history = pickle_idea2.dict2lst(d) - lst = [] - for word in request.form.getlist('marked'): - lst.append((word, [get_time()])) - d = pickle_idea2.merge_frequency(lst, lst_history) - if len(lst_history) > 999: - flash('You have way too many words in your difficult-words book. Delete some first.') - else: - pickle_idea2.save_frequency_to_pickle(d, user_freq_record) - flash('Added %s.' % (', '.join(request.form.getlist('marked')))) - return redirect(url_for('user_bp.userpage', username=username)) - else: - return 'Under construction' - -def get_time(): - ''' - 获取当前时间 - :return: 当前时间 - ''' - return datetime.now().strftime('%Y%m%d%H%M') # upper to minutes - +from datetime import datetime +from admin_service import ADMIN_NAME +from flask import * + +# from app import Yaml +# from app.Article import get_today_article, load_freq_history +# from app.WordFreq import WordFreq +# from app.wordfreqCMD import sort_in_descending_order + +import Yaml +from Article import get_today_article, load_freq_history +from WordFreq import WordFreq +from wordfreqCMD import sort_in_descending_order + +import pickle_idea +import pickle_idea2 + +import logging +logging.basicConfig(filename='log.txt', format='%(asctime)s %(message)s', level=logging.DEBUG) + +# 初始化蓝图 +userService = Blueprint("user_bp", __name__) + +path_prefix = '/var/www/wordfreq/wordfreq/' +path_prefix = './' # comment this line in deployment + +@userService.route("/get_next_article/",methods=['GET','POST']) +def get_next_article(username): + user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) + session['old_articleID'] = session.get('articleID') + if request.method == 'GET': + visited_articles = session.get("visited_articles") + if visited_articles['article_ids'][-1] == "null": # 如果当前还是“null”,则将“null”pop出来,无需index+=1 + visited_articles['article_ids'].pop() + else: # 当前不为“null”,直接 index+=1 + visited_articles["index"] += 1 + session["visited_articles"] = visited_articles + logging.debug('/get_next_article: start calling get_today_arcile()') + visited_articles, today_article, result_of_generate_article = get_today_article(user_freq_record, session.get('visited_articles')) + logging.debug('/get_next_arcile: done.') + data = { + 'visited_articles': visited_articles, + 'today_article': today_article, + 'result_of_generate_article': result_of_generate_article + } + else: + return 'Under construction' + return json.dumps(data) + +@userService.route("/get_pre_article/",methods=['GET']) +def get_pre_article(username): + user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) + if request.method == 'GET': + visited_articles = session.get("visited_articles") + if(visited_articles["index"]==0): + data='' + else: + visited_articles["index"] -= 1 # 上一篇,index-=1 + if visited_articles['article_ids'][-1] == "null": # 如果当前还是“null”,则将“null”pop出来 + visited_articles['article_ids'].pop() + session["visited_articles"] = visited_articles + visited_articles, today_article, result_of_generate_article = get_today_article(user_freq_record, session.get('visited_articles')) + data = { + 'visited_articles': visited_articles, + 'today_article': today_article, + 'result_of_generate_article':result_of_generate_article + } + return json.dumps(data) + +@userService.route("///unfamiliar", methods=['GET', 'POST']) +def unfamiliar(username, word): + ''' + + :param username: + :param word: + :return: + ''' + user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) + pickle_idea.unfamiliar(user_freq_record, word) + session['thisWord'] = word # 1. put a word into session + session['time'] = 1 + return "success" + + +@userService.route("///familiar", methods=['GET', 'POST']) +def familiar(username, word): + ''' + + :param username: + :param word: + :return: + ''' + user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) + pickle_idea.familiar(user_freq_record, word) + session['thisWord'] = word # 1. put a word into session + session['time'] = 1 + return "success" + + +@userService.route("///del", methods=['GET', 'POST']) +def deleteword(username, word): + ''' + 删除单词 + :param username: 用户名 + :param word: 单词 + :return: 重定位到用户界面 + ''' + user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) + pickle_idea2.deleteRecord(user_freq_record, word) + # 模板userpage_get.html中删除单词是异步执行,而flash的信息后续是同步执行的,所以注释这段代码;同时如果这里使用flash但不提取信息,则会影响 signup.html的显示。bug复现:删除单词后,点击退出,点击注册,注册页面就会出现提示信息 + # flash(f'{word} is no longer in your word list.') + return "success" + + +@userService.route("//userpage", methods=['GET', 'POST']) +def userpage(username): + ''' + 用户界面 + :param username: 用户名 + :return: 返回用户界面 + ''' + # 未登录,跳转到未登录界面 + if not session.get('logged_in'): + return render_template('not_login.html') + + # 用户过期 + user_expiry_date = session.get('expiry_date') + if datetime.now().strftime('%Y%m%d') > user_expiry_date: + return render_template('expiry.html', expiry_date=user_expiry_date) + + # 获取session里的用户名 + username = session.get('username') + + user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) + + if request.method == 'POST': # when we submit a form + content = request.form['content'] + f = WordFreq(content) + lst = f.get_freq() + return render_template('userpage_post.html',username=username,lst = lst, yml=Yaml.yml) + + elif request.method == 'GET': # when we load a html page + d = load_freq_history(user_freq_record) + lst = pickle_idea2.dict2lst(d) + lst2 = [] + for t in lst: + lst2.append((t[0], len(t[1]))) + lst3 = sort_in_descending_order(lst2) + words = '' + for x in lst3: + words += x[0] + ' ' + visited_articles, today_article, result_of_generate_article = get_today_article(user_freq_record, session.get('visited_articles')) + session['visited_articles'] = visited_articles + # 通过 today_article,加载前端的显示页面 + return render_template('userpage_get.html', + admin_name=ADMIN_NAME, + username=username, + session=session, + # flashed_messages=get_flashed_messages(), 仅有删除单词的时候使用到flash,而删除单词是异步执行,这里的信息提示是同步执行,所以就没有存在的必要了 + today_article=today_article, + result_of_generate_article=result_of_generate_article, + d_len=len(d), + lst3=lst3, + yml=Yaml.yml, + words=words) + +@userService.route("//mark", methods=['GET', 'POST']) +def user_mark_word(username): + ''' + 标记单词 + :param username: 用户名 + :return: 重定位到用户界面 + ''' + username = session[username] + user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) + if request.method == 'POST': + # 提交标记的单词 + d = load_freq_history(user_freq_record) + lst_history = pickle_idea2.dict2lst(d) + lst = [] + for word in request.form.getlist('marked'): + lst.append((word, [get_time()])) + d = pickle_idea2.merge_frequency(lst, lst_history) + if len(lst_history) > 999: + flash('You have way too many words in your difficult-words book. Delete some first.') + else: + + if pickle_idea2.save_frequency_to_pickle(d, user_freq_record): + flash('Added %s.' % (', '.join(request.form.getlist('marked')))) + else: + flash('%s.存在非法单词' % (', '.join(request.form.getlist('marked')))) + return redirect(url_for('user_bp.userpage', username=username)) + else: + return 'Under construction' + +def get_time(): + ''' + 获取当前时间 + :return: 当前时间 + ''' + return datetime.now().strftime('%Y%m%d%H%M') # upper to minutes +