badr_abdellaoui
/
EnglishPal
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor:pylint

Bug393-TanYanMei
倪玲丽 2023-06-04 00:35:43 +08:00
parent b22c654f0f
commit 3e0b8a0235
13 changed files with 259 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
app/Article.py
View File

@ -1,20 +1,18 @@
from WordFreq import WordFreq
from wordfreqCMD import youdao_link, sort_in_descending_order
from UseSqlite import InsertQuery, RecordQuery
import pickle_idea, pickle_idea2
import os import os
import random, glob import random
import hashlib import pickle_idea
from datetime import datetime
from flask import Flask, request, redirect, render_template, url_for, session, abort, flash, get_flashed_messages
from difficulty import get_difficulty_level_for_user, text_difficulty_level, user_difficulty_level from difficulty import get_difficulty_level_for_user, text_difficulty_level, user_difficulty_level
from UseSqlite import RecordQuery
path_prefix = '/var/www/wordfreq/wordfreq/' path_prefix = '/var/www/wordfreq/wordfreq/'
path_prefix = './' # comment this line in deployment path_prefix = './' # comment this line in deployment
def total_number_of_essays(): def total_number_of_essays():
'''
得到文章总数
return:文章数目
'''
rq = RecordQuery(path_prefix + 'static/wordfreqapp.db') rq = RecordQuery(path_prefix + 'static/wordfreqapp.db')
rq.instructions("SELECT * FROM article") rq.instructions("SELECT * FROM article")
rq.do() rq.do()
@ -23,16 +21,25 @@ def total_number_of_essays():
def get_article_title(s): def get_article_title(s):
'''
得到文章的标题
'''
return s.split('\n')[0] return s.split('\n')[0]
def get_article_body(s): def get_article_body(s):
'''
得到文章的内容
'''
lst = s.split('\n') lst = s.split('\n')
lst.pop(0) # remove the first line lst.pop(0) # remove the first line
return '\n'.join(lst) return '\n'.join(lst)
def get_today_article(user_word_list, visited_articles): def get_today_article(user_word_list, visited_articles):
'''
根据用户的单词列表和阅读过的文章返回需要的文章的全部信息
'''
rq = RecordQuery(path_prefix + 'static/wordfreqapp.db') rq = RecordQuery(path_prefix + 'static/wordfreqapp.db')
if visited_articles is None: if visited_articles is None:
visited_articles = { visited_articles = {
@ -42,10 +49,14 @@ def get_today_article(user_word_list, visited_articles):
if visited_articles["index"] > len(visited_articles["article_ids"])-1: # 生成新的文章，因此查找所有的文章 if visited_articles["index"] > len(visited_articles["article_ids"])-1: # 生成新的文章，因此查找所有的文章
rq.instructions("SELECT * FROM article") rq.instructions("SELECT * FROM article")
else: # 生成阅读过的文章，因此查询指定 article_id 的文章 else: # 生成阅读过的文章，因此查询指定 article_id 的文章
if visited_articles["article_ids"][visited_articles["index"]] == 'null': # 可能因为直接刷新页面导致直接去查询了'null'，因此当刷新的页面的时候，需要直接进行“上一篇”操作 # 可能因为直接刷新页面导致直接去查询了'null'，因此当刷新的页面的时候，需要直接进行“上一篇”操作
if visited_articles["article_ids"][visited_articles["index"]] == 'null':
visited_articles["index"] -= 1 visited_articles["index"] -= 1
visited_articles["article_ids"].pop() visited_articles["article_ids"].pop()
rq.instructions('SELECT * FROM article WHERE article_id=%d' % (visited_articles["article_ids"][visited_articles["index"]])) rq.instructions(
f'SELECT * FROM article WHERE article_id='
f'{visited_articles["article_ids"][visited_articles["index"]]}'
)
rq.do() rq.do()
result = rq.get_results() result = rq.get_results()
random.shuffle(result) random.shuffle(result)
@ -58,19 +69,23 @@ def get_today_article(user_word_list, visited_articles):
d = None d = None
result_of_generate_article = "not found" result_of_generate_article = "not found"
d_user = load_freq_history(user_word_list) d_user = load_freq_history(user_word_list)
user_level = user_difficulty_level(d_user, d3) # more consideration as user's behaviour is dynamic. Time factor should be considered. # 更多的考虑，因为用户的行为是动态的。应考虑时间因素。
user_level = user_difficulty_level(d_user, d3)
text_level = 0 text_level = 0
if visited_articles["index"] > len(visited_articles["article_ids"])-1: # 生成新的文章 if visited_articles["index"] > len(visited_articles["article_ids"])-1: # 生成新的文章
amount_of_visited_articles = len(visited_articles["article_ids"]) amount_of_visited_articles = len(visited_articles["article_ids"])
amount_of_existing_articles = result.__len__() amount_of_existing_articles = len(result)
if amount_of_visited_articles == amount_of_existing_articles: # 如果当前阅读过的文章的数量 == 存在的文章的数量，即所有的书本都阅读过了 # 如果当前阅读过的文章的数量 == 存在的文章的数量，即所有的书本都阅读过了
if amount_of_visited_articles == amount_of_existing_articles:
result_of_generate_article = "had read all articles" result_of_generate_article = "had read all articles"
else: else:
for k in range(3): # 最多尝试3次 for k in range(3): # 最多尝试3次
for reading in result: for reading in result:
text_level = text_difficulty_level(reading['text'], d3) text_level = text_difficulty_level(reading['text'], d3)
factor = random.gauss(0.8, 0.1) # a number drawn from Gaussian distribution with a mean of 0.8 and a stand deviation of 1 # 从高斯分布中得出的平均值为 0.8，站位偏差为 1 的数字
if reading['article_id'] not in visited_articles["article_ids"] and within_range(text_level, user_level, (8.0 - user_level) * factor): # 新的文章之前没有出现过且符合一定范围的水平 factor = random.gauss(0.8, 0.1)
# 新的文章之前没有出现过且符合一定范围的水平
if reading['article_id'] not in visited_articles["article_ids"] and within_range(text_level, user_level, (8.0 - user_level) * factor):
d = reading d = reading
visited_articles["article_ids"].append(d['article_id']) # 列表添加新的文章id；下面进行 visited_articles["article_ids"].append(d['article_id']) # 列表添加新的文章id；下面进行
result_of_generate_article = "found" result_of_generate_article = "found"
@ -87,8 +102,8 @@ def get_today_article(user_word_list, visited_articles):
today_article = None today_article = None
if d: if d:
today_article = { today_article = {
"user_level": '%4.2f' % user_level, "user_level": f'{user_level:4.2f}',
"text_level": '%4.2f' % text_level, "text_level": f'{text_level:4.2f}',
"date": d['date'], "date": d['date'],
"article_title": get_article_title(d['text']), "article_title": get_article_title(d['text']),
"article_body": get_article_body(d['text']), "article_body": get_article_body(d['text']),
@ -101,6 +116,9 @@ def get_today_article(user_word_list, visited_articles):
def load_freq_history(path): def load_freq_history(path):
'''
加载历史路径
'''
d = {} d = {}
if os.path.exists(path): if os.path.exists(path):
d = pickle_idea.load_record(path) d = pickle_idea.load_record(path)
@ -108,10 +126,16 @@ def load_freq_history(path):
def within_range(x, y, r): def within_range(x, y, r):
'''
判断x>y并且x-y<=r
'''
return x > y and abs(x - y) <= r return x > y and abs(x - y) <= r
def get_question_part(s): def get_question_part(s):
'''
得到问题部分
'''
s = s.strip() s = s.strip()
result = [] result = []
flag = 0 flag = 0
@ -128,6 +152,9 @@ def get_question_part(s):
def get_answer_part(s): def get_answer_part(s):
'''
得到答案部分
'''
s = s.strip() s = s.strip()
result = [] result = []
flag = 0 flag = 0

27
app/Login.py
View File

@ -1,8 +1,7 @@
import hashlib import hashlib
import string import string
from datetime import datetime, timedelta from datetime import datetime, timedelta
from UseSqlite import InsertQuery, RecordQuery from model.user import get_user_by_username, insert_user, update_password_by_username
def md5(s): def md5(s):
''' '''
MD5摘要 MD5摘要
@ -12,24 +11,33 @@ def md5(s):
h = hashlib.md5(s.encode(encoding='utf-8')) h = hashlib.md5(s.encode(encoding='utf-8'))
return h.hexdigest() return h.hexdigest()
# import model.user after the defination of md5(s) to avoid circular import
from model.user import get_user_by_username, insert_user, update_password_by_username
path_prefix = '/var/www/wordfreq/wordfreq/' path_prefix = '/var/www/wordfreq/wordfreq/'
path_prefix = './' # comment this line in deployment path_prefix = './' # comment this line in deployment
def verify_pass(newpass,oldpass): def verify_pass(newpass,oldpass):
if(newpass==oldpass): '''
判断新旧密码是否相同
'''
if newpass==oldpass:
return True return True
return False
def verify_user(username, password): def verify_user(username, password):
'''
判断用户名密码是否正确
'''
user = get_user_by_username(username) user = get_user_by_username(username)
encoded_password = md5(username + password) encoded_password = md5(username + password)
return user is not None and user.password == encoded_password return user is not None and user.password == encoded_password
def add_user(username, password): def add_user(username, password):
'''
添加新用户
'''
start_date = datetime.now().strftime('%Y%m%d') start_date = datetime.now().strftime('%Y%m%d')
expiry_date = (datetime.now() + timedelta(days=30)).strftime('%Y%m%d') # will expire after 30 days expiry_date = (datetime.now() + timedelta(days=30)).strftime('%Y%m%d') # will expire after 30 days
# 将用户名和密码一起加密，以免暴露不同用户的相同密码 # 将用户名和密码一起加密，以免暴露不同用户的相同密码
@ -38,6 +46,9 @@ def add_user(username, password):
def check_username_availability(username): def check_username_availability(username):
'''
检查用户名是否可用
'''
existed_user = get_user_by_username(username) existed_user = get_user_by_username(username)
return existed_user is None return existed_user is None
@ -60,10 +71,12 @@ def change_password(username, old_password, new_password):
def get_expiry_date(username): def get_expiry_date(username):
'''
获得过期日期
'''
user = get_user_by_username(username) user = get_user_by_username(username)
if user is None: if user is None:
return '20191024' return '20191024'
else:
return user.expiry_date return user.expiry_date
class UserName: class UserName:
@ -77,6 +90,7 @@ class UserName:
return 'Period (.) is not allowed as the first letter in the user name.' return 'Period (.) is not allowed as the first letter in the user name.'
if ' ' in self.username: # a user name must not include a whitespace if ' ' in self.username: # a user name must not include a whitespace
return 'Whitespace is not allowed in the user name.' return 'Whitespace is not allowed in the user name.'
for c in self.username: # a user name must not include special characters, except non-leading periods or underscores for c in self.username: # a user name must not include special characters, except non-leading periods or underscores
if c in string.punctuation and c != '.' and c != '_': if c in string.punctuation and c != '.' and c != '_':
return f'{c} is not allowed in the user name.' return f'{c} is not allowed in the user name.'
@ -92,4 +106,3 @@ class WarningMessage:
def __str__(self): def __str__(self):
return UserName(self.s).validate() return UserName(self.s).validate()

2
app/UseSqlite.py
View File

@ -13,7 +13,7 @@ class Sqlite3Template:
def __init__(self, db_fname): def __init__(self, db_fname):
self.db_fname = db_fname self.db_fname = db_fname
def connect(self, db_fname): def connect(self):
self.conn = sqlite3.connect(self.db_fname) self.conn = sqlite3.connect(self.db_fname)
def instructions(self, query_statement): def instructions(self, query_statement):

4
app/WordFreq.py
View File

@ -2,9 +2,9 @@
# Copyright 2019 (C) Hui Lan <hui.lan@cantab.net> # Copyright 2019 (C) Hui Lan <hui.lan@cantab.net>
# Written permission must be obtained from the author for commercial uses. # Written permission must be obtained from the author for commercial uses.
########################################################################### ###########################################################################
from wordfreqCMD import remove_punctuation, freq, sort_in_descending_order
import string import string
from wordfreqCMD import remove_punctuation, freq, sort_in_descending_order
class WordFreq: class WordFreq:
def __init__(self, s): def __init__(self, s):

1
app/Yaml.py
View File

@ -6,7 +6,6 @@ Yaml.py
./layout/partial/footer.html ./layout/partial/footer.html
''' '''
import yaml as YAML import yaml as YAML
import os
path_prefix = './' # comment this line in deployment path_prefix = './' # comment this line in deployment

43
app/account_service.py
View File

@ -1,45 +1,43 @@
from flask import * from flask import Blueprint, request, render_template, jsonify,session,redirect, url_for,escape
from Login import check_username_availability, verify_user, add_user,get_expiry_date, change_password, WarningMessage from Login import check_username_availability, verify_user, add_user,get_expiry_date, change_password, WarningMessage
# 初始化蓝图 # 初始化蓝图
accountService = Blueprint("accountService", __name__) accountService = Blueprint("accountService", __name__)
### Sign-up, login, logout ### ### sign-up, login, logout, reset###
@accountService.route("/signup", methods=['GET', 'POST']) @accountService.route("/signup", methods=['GET', 'POST'])
def signup(): def signup():
''' '''
注册 注册
:return: 根据注册是否成功返回不同界面 :return: 根据注册是否成功返回不同界面
''' '''
if request.method == 'GET':
# GET方法直接返回注册页面 # GET方法直接返回注册页面
if request.method == 'GET':
return render_template('signup.html') return render_template('signup.html')
elif request.method == 'POST':
# POST方法需判断是否注册成功，再根据结果返回不同的内容 # POST方法需判断是否注册成功，再根据结果返回不同的内容
username = escape(request.form['username']) username = escape(request.form['username'])
password = escape(request.form['password']) password = escape(request.form['password'])
#! 添加如下代码为了过滤注册时的非法字符 #! 添加如下代码为了过滤注册时的非法字符
warn = WarningMessage(username) warn = WarningMessage(username)
if str(warn) != 'OK': if str(warn) != 'OK':
return jsonify({'status': '3', 'warn': str(warn)}) return jsonify({'status': '3', 'warn': str(warn)})
available = check_username_availability(username) available = check_username_availability(username)
if not available: # 用户名不可用 # 用户名不可用
if not available:
return jsonify({'status': '0'}) return jsonify({'status': '0'})
else: # 添加账户信息 # 用户名可用,添加账户信息
add_user(username, password) add_user(username, password)
verified = verify_user(username, password) verified = verify_user(username, password)
# 注册成功，写入session
if verified: if verified:
# 写入session
session['logged_in'] = True session['logged_in'] = True
session[username] = username session[username] = username
session['username'] = username session['username'] = username
session['expiry_date'] = get_expiry_date(username) session['expiry_date'] = get_expiry_date(username)
session['visited_articles'] = None session['visited_articles'] = None
return jsonify({'status': '2'}) return jsonify({'status': '2'})
else: # 验证失败
return jsonify({'status': '1'}) return jsonify({'status': '1'})
@ -50,17 +48,16 @@ def login():
登录 登录
:return: 根据登录是否成功返回不同页面 :return: 根据登录是否成功返回不同页面
''' '''
# GET方法直接返回登录页面
if request.method == 'GET': if request.method == 'GET':
# GET请求
return render_template('login.html') return render_template('login.html')
elif request.method == 'POST':
# POST方法用于判断登录是否成功 # POST方法用于判断登录是否成功，检查数据库并验证用户，再根据结果返回不同的内容
# check database and verify user
username = escape(request.form['username']) username = escape(request.form['username'])
password = escape(request.form['password']) password = escape(request.form['password'])
verified = verify_user(username, password) verified = verify_user(username, password)
if verified:
# 登录成功，写入session # 登录成功，写入session
if verified:
session['logged_in'] = True session['logged_in'] = True
session[username] = username session[username] = username
session['username'] = username session['username'] = username
@ -68,7 +65,7 @@ def login():
session['expiry_date'] = user_expiry_date session['expiry_date'] = user_expiry_date
session['visited_articles'] = None session['visited_articles'] = None
return jsonify({'status': '1'}) return jsonify({'status': '1'})
else: #验证失败
return jsonify({'status': '0'}) return jsonify({'status': '0'})
@ -95,16 +92,18 @@ def reset():
username = session['username'] username = session['username']
if username == '': if username == '':
return redirect('/login') return redirect('/login')
if request.method == 'GET':
# GET请求返回修改密码页面 # GET请求返回修改密码页面
if request.method == 'GET':
return render_template('reset.html', username=session['username'], state='wait') return render_template('reset.html', username=session['username'], state='wait')
else:
# POST请求用于提交修改后信息 # POST请求用于提交修改后信息
old_password = escape(request.form['old-password']) old_password = escape(request.form['old-password'])
new_password = escape(request.form['new-password']) new_password = escape(request.form['new-password'])
flag = change_password(username, old_password, new_password) # flag表示是否修改成功 flag = change_password(username, old_password, new_password) # flag表示是否修改成功
# 修改成功
if flag: if flag:
session['logged_in'] = False session['logged_in'] = False
return jsonify({'status':'1'}) # 修改成功 return jsonify({'status':'1'})
else: # 修改失败
return jsonify({'status':'2'}) # 修改失败 return jsonify({'status':'2'})

58
app/admin_service.py
View File

@ -1,18 +1,22 @@
# System Library # System Library
from flask import * from flask import Blueprint, session, render_template, request, flash
# Personal library # Personal library
from Yaml import yml from Yaml import yml
from model.user import * from model.user import get_users, update_password_by_username
from model.article import * from model.user import update_expiry_time_by_username, get_user_by_username
from model.article import get_number_of_articles, get_page_articles
from model.article import delete_article_by_id, add_article
ADMIN_NAME = "lanhui" # unique admin name ADMIN_NAME = "lanhui" # unique admin name
_cur_page = 1 # current article page _cur_page = 1 # current article page
_page_size = 5 # article sizes per page _page_size = 5 # article sizes per page
adminService = Blueprint("admin_service", __name__) adminService = Blueprint("admin_service", __name__)
def check_is_admin(): def check_is_admin():
'''
判断是否是管理员登录
：return：不同页面和结果
'''
# 未登录，跳转到未登录界面 # 未登录，跳转到未登录界面
if not session.get("logged_in"): if not session.get("logged_in"):
return render_template("not_login.html") return render_template("not_login.html")
@ -26,6 +30,10 @@ def check_is_admin():
@adminService.route("/admin", methods=["GET"]) @adminService.route("/admin", methods=["GET"])
def admin(): def admin():
'''
判断是否是管理员登录
：return：不同页面
'''
is_admin = check_is_admin() is_admin = check_is_admin()
if is_admin != "pass": if is_admin != "pass":
return is_admin return is_admin
@ -37,6 +45,11 @@ def admin():
@adminService.route("/admin/article", methods=["GET", "POST"]) @adminService.route("/admin/article", methods=["GET", "POST"])
def article(): def article():
'''
管理文章
可添加文章、删除文章
return:不同页面
'''
global _cur_page, _page_size global _cur_page, _page_size
is_admin = check_is_admin() is_admin = check_is_admin()
@ -49,16 +62,15 @@ def article():
max(1, int(request.args.get("size", 5))), _article_number max(1, int(request.args.get("size", 5))), _article_number
) # 最小的size是1 ) # 最小的size是1
_cur_page = min( _cur_page = min(
max(1, int(request.args.get("page", 1))), _article_number // _page_size + (_article_number % _page_size > 0) max(1, int(request.args.get("page", 1))),
_article_number // _page_size + (_article_number % _page_size > 0)
) # 最小的page是1 ) # 最小的page是1
except ValueError: except ValueError:
return "page parmas must be int!" return "page parmas must be int!"
_articles = get_page_articles(_cur_page, _page_size) _articles = get_page_articles(_cur_page, _page_size)
for article in _articles: # 获取每篇文章的title for _article in _articles: # 获取每篇文章的title
article.title = article.text.split("\n")[0] _article.title = _article.text.split("\n")[0]
article.content = '<br/>'.join(article.text.split("\n")[1:]) _article.content = '<br/>'.join(_article.text.split("\n")[1:])
context = { context = {
"article_number": _article_number, "article_number": _article_number,
"text_list": _articles, "text_list": _articles,
@ -72,19 +84,19 @@ def article():
context["article_number"] = article_len context["article_number"] = article_len
context["text_list"] = get_page_articles(_cur_page, _page_size) context["text_list"] = get_page_articles(_cur_page, _page_size)
_articles = get_page_articles(_cur_page, _page_size) _articles = get_page_articles(_cur_page, _page_size)
for article in _articles: # 获取每篇文章的title for _article in _articles: # 获取每篇文章的title
article.title = article.text.split("\n")[0] _article.title = _article.text.split("\n")[0]
context["text_list"] = _articles context["text_list"] = _articles
if request.method == "GET": if request.method == "GET":
try: try:
delete_id = int(request.args.get("delete_id", 0)) delete_id = int(request.args.get("delete_id", 0))
except: except ValueError:
return "Delete article ID must be int!" return "Delete article ID must be int!"
if delete_id: # delete article if delete_id: # delete article
delete_article_by_id(delete_id) delete_article_by_id(delete_id)
_update_context() _update_context()
elif request.method == "POST": else:
data = request.form data = request.form
content = data.get("content", "") content = data.get("content", "")
source = data.get("source", "") source = data.get("source", "")
@ -102,10 +114,14 @@ def article():
@adminService.route("/admin/user", methods=["GET", "POST"]) @adminService.route("/admin/user", methods=["GET", "POST"])
def user(): def user():
'''
用户管理
可修改用户密码，过期日期
return：不同页面
'''
is_admin = check_is_admin() is_admin = check_is_admin()
if is_admin != "pass": if is_admin != "pass":
return is_admin return is_admin
context = { context = {
"user_list": get_users(), "user_list": get_users(),
"username": session.get("username"), "username": session.get("username"),
@ -127,6 +143,9 @@ def user():
@adminService.route("/admin/expiry", methods=["GET"]) @adminService.route("/admin/expiry", methods=["GET"])
def user_expiry_time(): def user_expiry_time():
'''
返回用户的过期日期
'''
is_admin = check_is_admin() is_admin = check_is_admin()
if is_admin != "pass": if is_admin != "pass":
return is_admin return is_admin
@ -135,8 +154,7 @@ def user_expiry_time():
if not username: if not username:
return "Username can't be empty." return "Username can't be empty."
user = get_user_by_username(username) existed_user = get_user_by_username(username)
if not user: if not existed_user:
return "User does not exist." return "User does not exist."
return existed_user.expiry_date
return user.expiry_date

25
app/difficulty.py
View File

@ -6,15 +6,17 @@
# Purpose: compute difficulty level of a English text # Purpose: compute difficulty level of a English text
import pickle import pickle
import math
from wordfreqCMD import remove_punctuation, freq, sort_in_descending_order, sort_in_ascending_order
import snowballstemmer import snowballstemmer
from wordfreqCMD import remove_punctuation, freq, sort_in_descending_order, sort_in_ascending_order
def load_record(pickle_fname): def load_record(pickle_fname):
f = open(pickle_fname, 'rb') '''
得到特定pickle文件中的数据
'''
with open(pickle_fname, 'rb') as f:
d = pickle.load(f) d = pickle.load(f)
f.close()
return d return d
@ -53,9 +55,7 @@ def get_difficulty_level_for_user(d1, d2):
stemmer = snowballstemmer.stemmer('english') stemmer = snowballstemmer.stemmer('english')
for k in d1: # 用户的词 for k in d1: # 用户的词
if k in d2: # 如果用户的词以原型的形式存在于词库d2中 if k not in d2: # 如果用户的词以原型的形式不存在于词库d2中
continue # 无需评级，跳过
else:
stem = stemmer.stemWord(k) stem = stemmer.stemWord(k)
if stem in d2: # 如果用户的词的词根存在于词库d2的词根库中 if stem in d2: # 如果用户的词的词根存在于词库d2的词根库中
d2[k] = d2[stem] # 按照词根进行评级 d2[k] = d2[stem] # 按照词根进行评级
@ -88,6 +88,9 @@ def revert_dict(d):
def user_difficulty_level(d_user, d): def user_difficulty_level(d_user, d):
'''
得到用户词汇的难度水平
'''
d_user2 = revert_dict(d_user) # key is date, and value is a list of words added in that date d_user2 = revert_dict(d_user) # key is date, and value is a list of words added in that date
count = 0 count = 0
geometric = 1 geometric = 1
@ -114,6 +117,9 @@ def user_difficulty_level(d_user, d):
def text_difficulty_level(s, d): def text_difficulty_level(s, d):
'''
得到用文章的难度水平
'''
s = remove_punctuation(s) s = remove_punctuation(s)
L = freq(s) L = freq(s)
@ -242,10 +248,7 @@ We need — for our farmers, our manufacturers, for, frankly, unions and non-uni
''' '''
# f = open('bbc-fulltext/bbc/entertainment/001.txt') # f = open('bbc-fulltext/bbc/entertainment/001.txt')
f = open('wordlist.txt') with open('wordlist.txt', encoding='utf-8') as f:
s = f.read() s = f.read()
f.close()
print(text_difficulty_level(s, d3)) print(text_difficulty_level(s, d3))

18
app/main.py
View File

@ -1,17 +1,24 @@
#! /usr/bin/python3 #! /usr/bin/python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import os
import random
########################################################################### ###########################################################################
# Copyright 2019 (C) Hui Lan <hui.lan@cantab.net> # Copyright 2019 (C) Hui Lan <hui.lan@cantab.net>
# Written permission must be obtained from the author for commercial uses. # Written permission must be obtained from the author for commercial uses.
########################################################################### ###########################################################################
from flask import escape import glob
from Login import * from flask import Flask, request, redirect, render_template, url_for, escape
from Article import * from WordFreq import WordFreq
import pickle_idea
from wordfreqCMD import sort_in_descending_order
from Article import load_freq_history, total_number_of_essays
import Yaml import Yaml
from user_service import userService from user_service import userService
from account_service import accountService from account_service import accountService
from admin_service import adminService, ADMIN_NAME from admin_service import adminService, ADMIN_NAME
app = Flask(__name__) app = Flask(__name__)
app.secret_key = 'lunch.time!' app.secret_key = 'lunch.time!'
@ -51,7 +58,6 @@ def appears_in_test(word, d):
''' '''
if not word in d: if not word in d:
return '' return ''
else:
return ','.join(d[word]) return ','.join(d[word])
@ -70,7 +76,7 @@ def mark_word():
d = pickle_idea.merge_frequency(lst, lst_history) d = pickle_idea.merge_frequency(lst, lst_history)
pickle_idea.save_frequency_to_pickle(d, path_prefix + 'static/frequency/frequency.p') pickle_idea.save_frequency_to_pickle(d, path_prefix + 'static/frequency/frequency.p')
return redirect(url_for('mainpage')) return redirect(url_for('mainpage'))
else: # 不回应GET请求 # 不回应GET请求
return 'Under construction' return 'Under construction'
@ -91,7 +97,7 @@ def mainpage():
pickle_idea.save_frequency_to_pickle(d, path_prefix + 'static/frequency/frequency.p') pickle_idea.save_frequency_to_pickle(d, path_prefix + 'static/frequency/frequency.p')
return render_template('mainpage_post.html', lst=lst, yml=Yaml.yml) return render_template('mainpage_post.html', lst=lst, yml=Yaml.yml)
elif request.method == 'GET': # when we load a html page if request.method == 'GET': # when we load a html page
random_ads = get_random_ads() random_ads = get_random_ads()
number_of_essays = total_number_of_essays() number_of_essays = total_number_of_essays()
d = load_freq_history(path_prefix + 'static/frequency/frequency.p') d = load_freq_history(path_prefix + 'static/frequency/frequency.p')

13
app/pickle_idea.py
View File

@ -19,7 +19,7 @@ def lst2dict(lst, d):
for x in lst: for x in lst:
word = x[0] word = x[0]
freq = x[1] freq = x[1]
if not word in d: if word not in d:
d[word] = freq d[word] = freq
else: else:
d[word] += freq d[word] += freq
@ -37,14 +37,14 @@ def merge_frequency(lst1, lst2):
def load_record(pickle_fname): def load_record(pickle_fname):
f = open(pickle_fname, 'rb') with open(pickle_fname, 'rb') as f:
d = pickle.load(f) pickle.load(f)
f.close() f.close()
return d return d
def save_frequency_to_pickle(d, pickle_fname): def save_frequency_to_pickle(d, pickle_fname):
f = open(pickle_fname, 'wb') with open(pickle_fname, 'wb') as f:
#exclusion_lst = ['one', 'no', 'has', 'had', 'do', 'that', 'have', 'by', 'not', 'but', 'we', 'this', 'my', 'him', 'so', 'or', 'as', 'are', 'it', 'from', 'with', 'be', 'can', 'for', 'an', 'if', 'who', 'whom', 'whose', 'which', 'the', 'to', 'a', 'of', 'and', 'you', 'i', 'he', 'she', 'they', 'me', 'was', 'were', 'is', 'in', 'at', 'on', 'their', 'his', 'her', 's', 'said', 'all', 'did', 'been', 'w'] #exclusion_lst = ['one', 'no', 'has', 'had', 'do', 'that', 'have', 'by', 'not', 'but', 'we', 'this', 'my', 'him', 'so', 'or', 'as', 'are', 'it', 'from', 'with', 'be', 'can', 'for', 'an', 'if', 'who', 'whom', 'whose', 'which', 'the', 'to', 'a', 'of', 'and', 'you', 'i', 'he', 'she', 'they', 'me', 'was', 'were', 'is', 'in', 'at', 'on', 'their', 'his', 'her', 's', 'said', 'all', 'did', 'been', 'w']
exclusion_lst = [] exclusion_lst = []
d2 = {} d2 = {}
@ -52,7 +52,6 @@ def save_frequency_to_pickle(d, pickle_fname):
if not k in exclusion_lst and not k.isnumeric() and len(k) > 1: if not k in exclusion_lst and not k.isnumeric() and len(k) > 1:
d2[k] = d[k] d2[k] = d[k]
pickle.dump(d2, f) pickle.dump(d2, f)
f.close()
def unfamiliar(path,word): def unfamiliar(path,word):
f = open(path,"rb") f = open(path,"rb")
@ -68,8 +67,8 @@ def familiar(path,word):
del dic[word][0] del dic[word][0]
else: else:
dic.pop(word) dic.pop(word)
fp = open(path,"wb") with open(path,"wb") as f:
pickle.dump(dic,fp) pickle.dump(dic,f)
if __name__ == '__main__': if __name__ == '__main__':

8
app/pickle_idea2.py
View File

@ -43,7 +43,7 @@ def dict2lst(d):
for k in d: for k in d:
lst.append((k, [datetime.now().strftime('%Y%m%d%H%M')])) lst.append((k, [datetime.now().strftime('%Y%m%d%H%M')]))
return lst return lst
elif isinstance(d[keys[0]], list): if isinstance(d[keys[0]], list):
return list(d.items()) # a list of (key, value) pairs return list(d.items()) # a list of (key, value) pairs
return [] return []
@ -56,21 +56,19 @@ def merge_frequency(lst1, lst2):
def load_record(pickle_fname): def load_record(pickle_fname):
f = open(pickle_fname, 'rb') with open(pickle_fname, 'rb') as f:
d = pickle.load(f) d = pickle.load(f)
f.close()
return d return d
def save_frequency_to_pickle(d, pickle_fname): def save_frequency_to_pickle(d, pickle_fname):
f = open(pickle_fname, 'wb') with open(pickle_fname, 'wb') as f:
exclusion_lst = ['one', 'no', 'has', 'had', 'do', 'that', 'have', 'by', 'not', 'but', 'we', 'this', 'my', 'him', 'so', 'or', 'as', 'are', 'it', 'from', 'with', 'be', 'can', 'for', 'an', 'if', 'who', 'whom', 'whose', 'which', 'the', 'to', 'a', 'of', 'and', 'you', 'i', 'he', 'she', 'they', 'me', 'was', 'were', 'is', 'in', 'at', 'on', 'their', 'his', 'her', 's', 'said', 'all', 'did', 'been', 'w'] exclusion_lst = ['one', 'no', 'has', 'had', 'do', 'that', 'have', 'by', 'not', 'but', 'we', 'this', 'my', 'him', 'so', 'or', 'as', 'are', 'it', 'from', 'with', 'be', 'can', 'for', 'an', 'if', 'who', 'whom', 'whose', 'which', 'the', 'to', 'a', 'of', 'and', 'you', 'i', 'he', 'she', 'they', 'me', 'was', 'were', 'is', 'in', 'at', 'on', 'their', 'his', 'her', 's', 'said', 'all', 'did', 'been', 'w']
d2 = {} d2 = {}
for k in d: for k in d:
if not k in exclusion_lst and not k.isnumeric() and not len(k) < 2: if not k in exclusion_lst and not k.isnumeric() and not len(k) < 2:
d2[k] = list(sorted(d[k])) # 原先这里是d2[k] = list(sorted(set(d[k]))) d2[k] = list(sorted(d[k])) # 原先这里是d2[k] = list(sorted(set(d[k])))
pickle.dump(d2, f) pickle.dump(d2, f)
f.close()

31
app/user_service.py
View File

@ -1,17 +1,10 @@
from datetime import datetime from datetime import datetime
from flask import render_template, session, url_for, redirect, request, json, escape, Blueprint
from admin_service import ADMIN_NAME from admin_service import ADMIN_NAME
from flask import *
# from app import Yaml
# from app.Article import get_today_article, load_freq_history
# from app.WordFreq import WordFreq
# from app.wordfreqCMD import sort_in_descending_order
import Yaml import Yaml
from Article import get_today_article, load_freq_history from Article import get_today_article, load_freq_history
from WordFreq import WordFreq from WordFreq import WordFreq
from wordfreqCMD import sort_in_descending_order from wordfreqCMD import sort_in_descending_order
import pickle_idea import pickle_idea
import pickle_idea2 import pickle_idea2
@ -23,7 +16,10 @@ path_prefix = './' # comment this line in deployment
@userService.route("/get_next_article/<username>",methods=['GET','POST']) @userService.route("/get_next_article/<username>",methods=['GET','POST'])
def get_next_article(username): def get_next_article(username):
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) '''
得到下一篇文章
'''
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_{username}.pickle'
session['old_articleID'] = session.get('articleID') session['old_articleID'] = session.get('articleID')
if request.method == 'GET': if request.method == 'GET':
visited_articles = session.get("visited_articles") visited_articles = session.get("visited_articles")
@ -44,7 +40,10 @@ def get_next_article(username):
@userService.route("/get_pre_article/<username>",methods=['GET']) @userService.route("/get_pre_article/<username>",methods=['GET'])
def get_pre_article(username): def get_pre_article(username):
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) '''
得到上一篇文章
'''
user_freq_record = path_prefix + 'static/frequency/' + f'frequency_{username}.pickle'
if request.method == 'GET': if request.method == 'GET':
visited_articles = session.get("visited_articles") visited_articles = session.get("visited_articles")
if(visited_articles["index"]==0): if(visited_articles["index"]==0):
@ -85,7 +84,7 @@ def familiar(username, word):
:param word: :param word:
:return: :return:
''' '''
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) user_freq_record = path_prefix + 'static/frequency/' + f'frequency_{username}.pickle'
pickle_idea.familiar(user_freq_record, word) pickle_idea.familiar(user_freq_record, word)
session['thisWord'] = word # 1. put a word into session session['thisWord'] = word # 1. put a word into session
session['time'] = 1 session['time'] = 1
@ -100,7 +99,7 @@ def deleteword(username, word):
:param word: 单词 :param word: 单词
:return: 重定位到用户界面 :return: 重定位到用户界面
''' '''
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) user_freq_record = path_prefix + 'static/frequency/' + f'frequency_{username}.pickle'
pickle_idea2.deleteRecord(user_freq_record, word) pickle_idea2.deleteRecord(user_freq_record, word)
# 模板userpage_get.html中删除单词是异步执行，而flash的信息后续是同步执行的，所以注释这段代码；同时如果这里使用flash但不提取信息，则会影响 signup.html的显示。bug复现：删除单词后，点击退出，点击注册，注册页面就会出现提示信息 # 模板userpage_get.html中删除单词是异步执行，而flash的信息后续是同步执行的，所以注释这段代码；同时如果这里使用flash但不提取信息，则会影响 signup.html的显示。bug复现：删除单词后，点击退出，点击注册，注册页面就会出现提示信息
# flash(f'{word} is no longer in your word list.') # flash(f'{word} is no longer in your word list.')
@ -126,7 +125,7 @@ def userpage(username):
# 获取session里的用户名 # 获取session里的用户名
username = session.get('username') username = session.get('username')
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) user_freq_record = path_prefix + 'static/frequency/' + f'frequency_{username}.pickle'
if request.method == 'POST': # when we submit a form if request.method == 'POST': # when we submit a form
content = escape(request.form['content']) content = escape(request.form['content'])
@ -134,7 +133,7 @@ def userpage(username):
lst = f.get_freq() lst = f.get_freq()
return render_template('userpage_post.html',username=username,lst = lst, yml=Yaml.yml) return render_template('userpage_post.html',username=username,lst = lst, yml=Yaml.yml)
elif request.method == 'GET': # when we load a html page if request.method == 'GET': # when we load a html page
d = load_freq_history(user_freq_record) d = load_freq_history(user_freq_record)
lst = pickle_idea2.dict2lst(d) lst = pickle_idea2.dict2lst(d)
lst2 = [] lst2 = []
@ -167,7 +166,7 @@ def user_mark_word(username):
:return: 重定位到用户界面 :return: 重定位到用户界面
''' '''
username = session[username] username = session[username]
user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) user_freq_record = path_prefix + 'static/frequency/' + f'frequency_{username}.pickle'
if request.method == 'POST': if request.method == 'POST':
# 提交标记的单词 # 提交标记的单词
d = load_freq_history(user_freq_record) d = load_freq_history(user_freq_record)
@ -178,7 +177,6 @@ def user_mark_word(username):
d = pickle_idea2.merge_frequency(lst, lst_history) d = pickle_idea2.merge_frequency(lst, lst_history)
pickle_idea2.save_frequency_to_pickle(d, user_freq_record) pickle_idea2.save_frequency_to_pickle(d, user_freq_record)
return redirect(url_for('user_bp.userpage', username=username)) return redirect(url_for('user_bp.userpage', username=username))
else:
return 'Under construction' return 'Under construction'
def get_time(): def get_time():
@ -187,4 +185,3 @@ def get_time():
:return: 当前时间 :return: 当前时间
''' '''
return datetime.now().strftime('%Y%m%d%H%M') # upper to minutes return datetime.now().strftime('%Y%m%d%H%M') # upper to minutes

24
app/wordfreqCMD.py
View File

@ -5,8 +5,8 @@
import collections import collections
import string import string
import operator import os
import os, sys # 引入模块sys，因为我要用里面的sys.argv列表中的信息来读取命令行参数。 import sys # 引入模块sys，因为我要用里面的sys.argv列表中的信息来读取命令行参数。
import pickle_idea import pickle_idea
def freq(fruit): def freq(fruit):
@ -32,14 +32,16 @@ def youdao_link(s): # 有道链接
def file2str(fname):#文件转字符 def file2str(fname):#文件转字符
f = open(fname) #打开 with open(fname, encoding="utf-8") as f:
s = f.read() #读取 s = f.read()
f.close() #关闭
return s return s
def remove_punctuation(s): # 这里是s是形参 (parameter)。函数被调用时才给s赋值。 def remove_punctuation(s): # 这里是s是形参 (parameter)。函数被调用时才给s赋值。
special_characters = '\_©~<=>+/[]*&$%^@.,?!:;#()"“”—‘’{}|' # 把里面的字符都去掉 '''
删除文本里的标点符号
'''
special_characters = r'\_©~<=>+/[]*&$%^@.,?!:;#()"“”—‘’{}|' # 把里面的字符都去掉
for c in special_characters: for c in special_characters:
s = s.replace(c, ' ') # 防止出现把 apple,apple 移掉逗号后变成 appleapple 情况 s = s.replace(c, ' ') # 防止出现把 apple,apple 移掉逗号后变成 appleapple 情况
s = s.replace('--', ' ') s = s.replace('--', ' ')
@ -78,11 +80,10 @@ def make_html_page(lst, fname): # 只是在wordfreqCMD.py中的main函数中调
count = 1 count = 1
for x in lst: for x in lst:
# <a href="">word</a> # <a href="">word</a>
s += '<p>%d <a href="%s">%s</a> (%d)</p>' % (count, youdao_link(x[0]), x[0], x[1]) s += f'<p>{count} <a href="{youdao_link(x[0])}">{x[0]}</a> ({x[1]})</p>'
count += 1 count += 1
f = open(fname, 'w') with open(fname, 'w', encoding="utf-8") as f:
f.write(s) f.write(s)
f.close()
## main（程序入口） ## main（程序入口）
@ -101,7 +102,7 @@ if __name__ == '__main__':
s = remove_punctuation(s) # 这里是s是实参(argument)，里面有值 s = remove_punctuation(s) # 这里是s是实参(argument)，里面有值
L = freq(s) L = freq(s)
for x in sort_in_descending_order(L): for x in sort_in_descending_order(L):
print('%s\t%d\t%s' % (x[0], x[1], youdao_link(x[0])))#函数导出 print(f'{x[0]}\t{x[1]}\t{youdao_link(x[0])}' )#函数导出
# 把频率的结果放result.html中 # 把频率的结果放result.html中
make_html_page(sort_in_descending_order(L), 'result.html') make_html_page(sort_in_descending_order(L), 'result.html')
@ -118,6 +119,3 @@ if __name__ == '__main__':
lst_history = pickle_idea.dict2lst(d) lst_history = pickle_idea.dict2lst(d)
d = pickle_idea.merge_frequency(L, lst_history) d = pickle_idea.merge_frequency(L, lst_history)
pickle_idea.save_frequency_to_pickle(d, 'frequency.p') pickle_idea.save_frequency_to_pickle(d, 'frequency.p')