diff --git a/.gitignore b/.gitignore index 3d901ba..33f789d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,12 +2,20 @@ venv/ app/__init__.py app/__pycache__/ +.DS_Store +app/.DS_Store app/sqlite_commands.py app/static/usr/*.jpg app/static/img/ app/static/frequency/frequency_*.pickle app/static/frequency/frequency.p -app/static/wordfreqapp.db +app/wordfreqapp.db +app/db/wordfreqapp.db app/static/donate-the-author.jpg app/static/donate-the-author-hidden.jpg -app/model/__pycache__/ \ No newline at end of file +app/model/__pycache__/ +app/test/__pycache__/ +app/test/.pytest_cache/ +app/test/pytest_report.html +app/test/assets +app/log.txt diff --git a/Dockerfile b/Dockerfile index 284195a..55e5946 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM tiangolo/uwsgi-nginx-flask:python3.6 -COPY requirements.txt /app -RUN pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ -COPY ./app /app +FROM tiangolo/uwsgi-nginx-flask:python3.8-alpine +COPY requirements.txt /tmp +COPY ./app/ /app/ +RUN pip3 install -U pip -i https://mirrors.aliyun.com/pypi/simple/ +RUN pip3 install -r /tmp/requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ diff --git a/Jenkinsfile b/Jenkinsfile index 2633859..c3772cc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,8 +10,8 @@ pipeline { stages { stage('MakeDatabasefile') { steps { - sh 'touch ./app/static/wordfreqapp.db && rm -f ./app/static/wordfreqapp.db' - sh 'cat ./app/static/wordfreqapp.sql | sqlite3 ./app/static/wordfreqapp.db' + sh 'touch ./app/wordfreqapp.db && rm -f ./app/wordfreqapp.db' + sh 'cat ./app/static/wordfreqapp.sql | sqlite3 ./app/wordfreqapp.db' } } stage('BuildIt') { diff --git a/README.md b/README.md index 14cc9aa..15fc966 100644 --- a/README.md +++ b/README.md @@ -61,15 +61,15 @@ My steps for deploying English on a Ubuntu server. All articles are stored in the `article` table in a SQLite file called -`app/static/wordfreqapp.db`. +`app/db/wordfreqapp.db`. ### Adding new articles -To add articles, open and edit `app/static/wordfreqapp.db` using DB Browser for SQLite (https://sqlitebrowser.org). +To add articles, open and edit `app/db/wordfreqapp.db` using DB Browser for SQLite (https://sqlitebrowser.org). ### Extending an account's expiry date -By default, an account's expiry is 30 days after first sign-up. To extend account's expiry date, open and edit `user` table in `app/static/wordfreqapp.db`. Simply update field `expiry_date`. +By default, an account's expiry is 30 days after first sign-up. To extend account's expiry date, open and edit `user` table in `app/db/wordfreqapp.db`. Simply update field `expiry_date`. ### Exporting the database @@ -95,7 +95,7 @@ sqlite3 wordfreqapp.db`. Delete wordfreqapp.db first if it exists. ### Uploading wordfreqapp.db to the server -`pscp wordfreqapp.db lanhui@118.*.*.118:/home/lanhui/englishpal2/EnglishPal/app/static` +`pscp wordfreqapp.db lanhui@118.*.*.118:/home/lanhui/englishpal2/EnglishPal/app/db/` @@ -129,6 +129,28 @@ We welcome feedback on EnglishPal. Feedback examples: EnglishPal's bugs and improvement suggestions are recorded in [Bugzilla](http://118.25.96.118/bugzilla/buglist.cgi?bug_status=__all__&list_id=1302&order=Importance&product=EnglishPal&query_format=specific). Send (lanhui at zjnu.edu.cn) an email message for opening a Bugzilla account or reporting a bug. +## End-to-end testing + +We use the Selenium test framework to test our app. + +In order to run the test, first we need to download a webdriver executable. + +Microsoft Edge's webdriver can be downloaded from [microsoft-edge-tools-webdriver](https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/). Make sure the version we download matches the version of the web browser installed on our laptop. + +After extracting the downloaded zip file (e.g., edgedriver_win64.zip), rename msedgedriver.exe to MicrosoftWebDriver.exe. + +Add MicrosoftWebDriver.exe's path to system's PATH variable. + +Install the following dependencies too: + +- pip install -U selenium==3.141.0 +- pip install -U urllib3==1.26.2 + +Run English Pal first, then run the test using pytest as follows: pytest --html=pytest_report.html test_add_word.py + +The above command will generate a HTML report file pytest_report.html after finishing executing test_add_word.py. Note: you need to install pytest-html package first: pip install pytest-html. + +You may also want to use [webdriver-manager](https://pypi.org/project/webdriver-manager/) from PyPI, so that you can avoid tediously installing a web driver executable manually. However, my experience shows that webdriver-manager is too slow. For example, it took me 16 minutes to run 9 tests, while with the pre-installed web driver executable, it took less than 2 minutes. ## TODO diff --git a/app/Article.py b/app/Article.py index df9ac3a..566ceb6 100644 --- a/app/Article.py +++ b/app/Article.py @@ -1,6 +1,5 @@ from WordFreq import WordFreq from wordfreqCMD import youdao_link, sort_in_descending_order -from UseSqlite import InsertQuery, RecordQuery import pickle_idea, pickle_idea2 import os import random, glob @@ -8,18 +7,15 @@ import hashlib from datetime import datetime from flask import Flask, request, redirect, render_template, url_for, session, abort, flash, get_flashed_messages from difficulty import get_difficulty_level_for_user, text_difficulty_level, user_difficulty_level +from model.article import get_all_articles, get_article_by_id, get_number_of_articles +import logging - -path_prefix = '/var/www/wordfreq/wordfreq/' -path_prefix = './' # comment this line in deployment +path_prefix = './' +db_path_prefix = './db/' # comment this line in deployment def total_number_of_essays(): - rq = RecordQuery(path_prefix + 'static/wordfreqapp.db') - rq.instructions("SELECT * FROM article") - rq.do() - result = rq.get_results() - return len(result) + return get_number_of_articles() def get_article_title(s): @@ -33,32 +29,36 @@ def get_article_body(s): def get_today_article(user_word_list, visited_articles): - rq = RecordQuery(path_prefix + 'static/wordfreqapp.db') if visited_articles is None: visited_articles = { "index" : 0, # 为 article_ids 的索引 "article_ids": [] # 之前显示文章的id列表，越后越新 } if visited_articles["index"] > len(visited_articles["article_ids"])-1: # 生成新的文章，因此查找所有的文章 - rq.instructions("SELECT * FROM article") + result = get_all_articles() else: # 生成阅读过的文章，因此查询指定 article_id 的文章 if visited_articles["article_ids"][visited_articles["index"]] == 'null': # 可能因为直接刷新页面导致直接去查询了'null'，因此当刷新的页面的时候，需要直接进行“上一篇”操作 visited_articles["index"] -= 1 visited_articles["article_ids"].pop() - rq.instructions('SELECT * FROM article WHERE article_id=%d' % (visited_articles["article_ids"][visited_articles["index"]])) - rq.do() - result = rq.get_results() + article_id = visited_articles["article_ids"][visited_articles["index"]] + result = get_article_by_id(article_id) random.shuffle(result) # Choose article according to reader's level - d1 = load_freq_history(path_prefix + 'static/frequency/frequency.p') + logging.debug('* get_today_article(): start d1 = ... ') + d1 = load_freq_history(user_word_list) d2 = load_freq_history(path_prefix + 'static/words_and_tests.p') + logging.debug(' ... get_today_article(): get_difficulty_level_for_user() start') d3 = get_difficulty_level_for_user(d1, d2) + logging.debug(' ... get_today_article(): done') d = None result_of_generate_article = "not found" + d_user = load_freq_history(user_word_list) + logging.debug('* get_today_article(): user_difficulty_level() start') user_level = user_difficulty_level(d_user, d3) # more consideration as user's behaviour is dynamic. Time factor should be considered. + logging.debug('* get_today_article(): done') text_level = 0 if visited_articles["index"] > len(visited_articles["article_ids"])-1: # 生成新的文章 amount_of_visited_articles = len(visited_articles["article_ids"]) @@ -87,8 +87,8 @@ def get_today_article(user_word_list, visited_articles): today_article = None if d: today_article = { - "user_level": '%4.2f' % user_level, - "text_level": '%4.2f' % text_level, + "user_level": '%4.1f' % user_level, + "text_level": '%4.1f' % text_level, "date": d['date'], "article_title": get_article_title(d['text']), "article_body": get_article_body(d['text']), diff --git a/app/Login.py b/app/Login.py index cd750d1..17d92fa 100644 --- a/app/Login.py +++ b/app/Login.py @@ -1,7 +1,6 @@ import hashlib import string from datetime import datetime, timedelta -from UseSqlite import InsertQuery, RecordQuery def md5(s): ''' diff --git a/app/UseSqlite.py b/app/UseSqlite.py deleted file mode 100644 index ea4baeb..0000000 --- a/app/UseSqlite.py +++ /dev/null @@ -1,87 +0,0 @@ -########################################################################### -# Copyright 2019 (C) Hui Lan -# Written permission must be obtained from the author for commercial uses. -########################################################################### - - -# Reference: Dusty Phillips. Python 3 Objected-oriented Programming Second Edition. Pages 326-328. -# Copyright (C) 2019 Hui Lan - -import sqlite3 - -class Sqlite3Template: - def __init__(self, db_fname): - self.db_fname = db_fname - - def connect(self, db_fname): - self.conn = sqlite3.connect(self.db_fname) - - def instructions(self, query_statement): - raise NotImplementedError() - - def operate(self): - self.conn.row_factory = sqlite3.Row - self.results = self.conn.execute(self.query) # self.query is to be given in the child classes - self.conn.commit() - - def format_results(self): - raise NotImplementedError() - - def do(self): - self.connect(self.db_fname) - self.instructions(self.query) - self.operate() - - def instructions_with_parameters(self, query_statement, parameters): - self.query = query_statement - self.parameters = parameters - - def do_with_parameters(self): - self.connect(self.db_fname) - self.instructions_with_parameters(self.query, self.parameters) - self.operate_with_parameters() - - def operate_with_parameters(self): - self.conn.row_factory = sqlite3.Row - self.results = self.conn.execute(self.query, self.parameters) # self.query is to be given in the child classes - self.conn.commit() - - -class InsertQuery(Sqlite3Template): - def instructions(self, query): - self.query = query - - -class RecordQuery(Sqlite3Template): - def instructions(self, query): - self.query = query - - def format_results(self): - output = [] - for row_dict in self.results.fetchall(): - lst = [] - for k in dict(row_dict): - lst.append( row_dict[k] ) - output.append(', '.join(lst)) - return '\n\n'.join(output) - - def get_results(self): - result = [] - for row_dict in self.results.fetchall(): - result.append( dict(row_dict) ) - return result - - - -if __name__ == '__main__': - - #iq = InsertQuery('RiskDB.db') - #iq.instructions("INSERT INTO inspection Values ('FoodSupplies', 'RI2019051301', '2019-05-13', '{}')") - #iq.do() - #iq.instructions("INSERT INTO inspection Values ('CarSupplies', 'RI2019051302', '2019-05-13', '{[{\"risk_name\":\"elevator\"}]}')") - #iq.do() - - rq = RecordQuery('wordfreqapp.db') - rq.instructions("SELECT * FROM article WHERE level=3") - rq.do() - #print(rq.format_results()) diff --git a/app/account_service.py b/app/account_service.py index a7ed0c4..fd5f7f6 100644 --- a/app/account_service.py +++ b/app/account_service.py @@ -1,4 +1,5 @@ from flask import * +from markupsafe import escape from Login import check_username_availability, verify_user, add_user, get_expiry_date, change_password, WarningMessage diff --git a/app/admin_service.py b/app/admin_service.py index a604b5e..c461af9 100644 --- a/app/admin_service.py +++ b/app/admin_service.py @@ -1,5 +1,6 @@ # System Library from flask import * +from markupsafe import escape # Personal library from Yaml import yml @@ -37,6 +38,22 @@ def admin(): @adminService.route("/admin/article", methods=["GET", "POST"]) def article(): + + def _make_title_and_content(article_lst): + for article in article_lst: + text = escape(article.text) # Fix XSS vulnerability, contributed by Xu Xuan + article.title = text.split("\n")[0] + article.content = '
'.join(text.split("\n")[1:]) + + + def _update_context(): + article_len = get_number_of_articles() + context["article_number"] = article_len + context["text_list"] = get_page_articles(_cur_page, _page_size) + _articles = get_page_articles(_cur_page, _page_size) + _make_title_and_content(_articles) + context["text_list"] = _articles + global _cur_page, _page_size is_admin = check_is_admin() @@ -44,20 +61,15 @@ def article(): return is_admin _article_number = get_number_of_articles() + try: - _page_size = min( - max(1, int(request.args.get("size", 5))), _article_number - ) # 最小的size是1 - _cur_page = min( - max(1, int(request.args.get("page", 1))), _article_number // _page_size + (_article_number % _page_size > 0) - ) # 最小的page是1 + _page_size = min(max(1, int(request.args.get("size", 5))), _article_number) # 最小的size是1 + _cur_page = min(max(1, int(request.args.get("page", 1))), _article_number // _page_size + (_article_number % _page_size > 0)) # 最小的page是1 except ValueError: - return "page parmas must be int!" - + return "page parameters must be integer!" + _articles = get_page_articles(_cur_page, _page_size) - for article in _articles: # 获取每篇文章的title - article.title = article.text.split("\n")[0] - article.content = '
'.join(article.text.split("\n")[1:]) + _make_title_and_content(_articles) context = { "article_number": _article_number, @@ -67,23 +79,16 @@ def article(): "username": session.get("username"), } - def _update_context(): - article_len = get_number_of_articles() - context["article_number"] = article_len - context["text_list"] = get_page_articles(_cur_page, _page_size) - _articles = get_page_articles(_cur_page, _page_size) - for article in _articles: # 获取每篇文章的title - article.title = article.text.split("\n")[0] - context["text_list"] = _articles if request.method == "GET": try: delete_id = int(request.args.get("delete_id", 0)) except: - return "Delete article ID must be int!" + return "Delete article ID must be integer!" if delete_id: # delete article delete_article_by_id(delete_id) _update_context() + elif request.method == "POST": data = request.form content = data.get("content", "") @@ -97,6 +102,7 @@ def article(): _update_context() title = content.split('\n')[0] flash(f'Article added. Title: {title}') + return render_template("admin_manage_article.html", **context) diff --git a/app/db/README.txt b/app/db/README.txt new file mode 100644 index 0000000..bb826a6 --- /dev/null +++ b/app/db/README.txt @@ -0,0 +1 @@ +Put wordfreqapp.db here diff --git a/app/difficulty.py b/app/difficulty.py index cb93768..1bd8d68 100644 --- a/app/difficulty.py +++ b/app/difficulty.py @@ -18,6 +18,7 @@ def load_record(pickle_fname): return d +ENGLISH_WORD_DIFFICULTY_DICT = {} def convert_test_type_to_difficulty_level(d): """ 对原本的单词库中的单词进行难度评级 @@ -39,8 +40,10 @@ def convert_test_type_to_difficulty_level(d): elif 'BBC' in d[k]: result[k] = 8 - return result # {'apple': 4, ...} + global ENGLISH_WORD_DIFFICULTY_DICT + ENGLISH_WORD_DIFFICULTY_DICT = result + return result # {'apple': 4, ...} def get_difficulty_level_for_user(d1, d2): """ @@ -49,7 +52,11 @@ def get_difficulty_level_for_user(d1, d2): 在d2的后面添加单词，没有新建一个新的字典 """ # TODO: convert_test_type_to_difficulty_level() should not be called every time. Each word's difficulty level should be pre-computed. - d2 = convert_test_type_to_difficulty_level(d2) # 根据d2的标记评级{'apple': 4, 'abandon': 4, ...} + if ENGLISH_WORD_DIFFICULTY_DICT == {}: + d2 = convert_test_type_to_difficulty_level(d2) # 根据d2的标记评级{'apple': 4, 'abandon': 4, ...} + else: + d2 = ENGLISH_WORD_DIFFICULTY_DICT + stemmer = snowballstemmer.stemmer('english') for k in d1: # 用户的词 diff --git a/app/main.py b/app/main.py index 4e3f829..19bd889 100644 --- a/app/main.py +++ b/app/main.py @@ -1,19 +1,19 @@ -#! /usr/bin/python3 -# -*- coding: utf-8 -*- - ########################################################################### # Copyright 2019 (C) Hui Lan # Written permission must be obtained from the author for commercial uses. ########################################################################### -from flask import escape +from flask import abort +from markupsafe import escape from Login import * from Article import * import Yaml from user_service import userService from account_service import accountService from admin_service import adminService, ADMIN_NAME +import os + app = Flask(__name__) -app.secret_key = 'lunch.time!' +app.secret_key = os.urandom(32) # 将蓝图注册到Lab app app.register_blueprint(userService) @@ -54,7 +54,6 @@ def appears_in_test(word, d): else: return ','.join(d[word]) - @app.route("/mark", methods=['GET', 'POST']) def mark_word(): ''' diff --git a/app/model/__init__.py b/app/model/__init__.py index 9526313..f5256a2 100644 --- a/app/model/__init__.py +++ b/app/model/__init__.py @@ -1,7 +1,7 @@ from pony.orm import * db = Database() -db.bind("sqlite", "../static/wordfreqapp.db", create_db=True) # bind sqlite file +db.bind("sqlite", "../db/wordfreqapp.db", create_db=True) # bind sqlite file class User(db.Entity): diff --git a/app/model/article.py b/app/model/article.py index a3b4bf7..bf19ded 100644 --- a/app/model/article.py +++ b/app/model/article.py @@ -7,7 +7,7 @@ def add_article(content, source="manual_input", level="5", question="No question Article( text=content, source=source, - date=datetime.now().strftime("%-d %b %Y"), # format style of `5 Oct 2022` + date=datetime.now().strftime("%d %b %Y"), # format style of `5 Oct 2022` level=level, question=question, ) @@ -32,3 +32,17 @@ def get_page_articles(num, size): x for x in Article.select().order_by(desc(Article.article_id)).page(num, size) ] + + +def get_all_articles(): + articles = [] + with db_session: + for article in Article.select(): + articles.append(article.to_dict()) + return articles + + +def get_article_by_id(article_id): + with db_session: + article = Article.get(article_id=article_id) + return [article.to_dict()] diff --git a/app/static/config.yml b/app/static/config.yml index 285f31f..173bc6b 100644 --- a/app/static/config.yml +++ b/app/static/config.yml @@ -9,6 +9,7 @@ js: - ../static/js/jquery.js - ../static/js/read.js - ../static/js/word_operation.js + - ../static/js/checkboxes.js bottom: # 在页面加载完之后加载 - ../static/js/fillword.js - ../static/js/highlight.js diff --git a/app/static/js/checkboxes.js b/app/static/js/checkboxes.js new file mode 100644 index 0000000..297df55 --- /dev/null +++ b/app/static/js/checkboxes.js @@ -0,0 +1,5 @@ +function toggleCheckboxSelection(checkStatus) { + // used in userpage_post.html + const checkBoxes = document.getElementsByName('marked'); + checkBoxes.forEach((checkbox) => { checkbox.checked = checkStatus;} ); +} diff --git a/app/static/js/fillword.js b/app/static/js/fillword.js index b967633..b3a8b42 100644 --- a/app/static/js/fillword.js +++ b/app/static/js/fillword.js @@ -29,3 +29,8 @@ function onReadClick() { function onChooseClick() { isChoose = !isChoose; } + +// 如果网页刷新，停止播放声音 +if (performance.getEntriesByType("navigation")[0].type == "reload") { + Reader.stopRead(); +} \ No newline at end of file diff --git a/app/static/js/highlight.js b/app/static/js/highlight.js index 0cea31a..76b4793 100644 --- a/app/static/js/highlight.js +++ b/app/static/js/highlight.js @@ -22,62 +22,38 @@ function getWord() { function highLight() { if (!isHighlight) return; - let articleContent = document.getElementById("article").innerText; //将原来的.innerText改为.innerHtml，使用innerText会把原文章中所包含的
标签去除，导致处理后的文章内容失去了原来的格式 + let articleContent = document.getElementById("article").innerHTML; // innerHTML保留HTML标签来保持部分格式，且适配不同的浏览器 let pickedWords = document.getElementById("selected-words"); // words picked to the text area let dictionaryWords = document.getElementById("selected-words2"); // words appearing in the user's new words list - let allWords = ""; //初始化allWords的值，避免进入判断后编译器认为allWords未初始化的问题 - if(dictionaryWords != null){//增加一个判断，检查生词本里面是否为空，如果为空，allWords只添加选中的单词 - allWords = pickedWords.value + " " + dictionaryWords.value; - } - else{ - allWords = pickedWords.value + " "; - } - const list = allWords.split(" ");//将所有的生词放入一个list中，用于后续处理 + let allWords = dictionaryWords === null ? pickedWords.value + " " : pickedWords.value + " " + dictionaryWords.value; + const list = allWords.split(" "); // 将所有的生词放入一个list中 + let totalSet = new Set(); for (let i = 0; i < list.length; ++i) { - list[i] = list[i].replace(/(^\s*)|(\s*$)/g, ""); //消除单词两边的空字符 - list[i] = list[i].replace('|', ""); - list[i] = list[i].replace('?', ""); - if (list[i] !== "" && "".indexOf(list[i]) === -1 && "".indexOf(list[i]) === -1) { - //将文章中所有出现该单词word的地方改为："" + word + ""。 正则表达式RegExp()中，"\\b"代表单词边界匹配。 - - //修改代码 - let articleContent_fb = articleContent; //文章副本 - while(articleContent_fb.toLowerCase().indexOf(list[i].toLowerCase()) !== -1 && list[i]!=""){ - //找到副本中和list[i]匹配的第一个单词(第一种匹配情况),并赋值给list[i]。 - const index = articleContent_fb.toLowerCase().indexOf(list[i].toLowerCase()); - list[i] = articleContent_fb.substring(index, index + list[i].length); - - articleContent_fb = articleContent_fb.substring(index + list[i].length); // 使用副本中list[i]之后的子串替换掉副本 - articleContent = articleContent.replace(new RegExp("\\b"+list[i]+"\\b","g"),"" + list[i] + ""); + list[i] = list[i].replace(/(^\W*)|(\W*$)/g, ""); // 消除单词两边的非单词字符 + if (list[i] != "" && !totalSet.has(list[i])) { + // 返回所有匹配单词的集合, 正则表达式RegExp()中, "\b"匹配一个单词的边界, g 表示全局匹配, i 表示对大小写不敏感。 + let matches = new Set(articleContent.match(new RegExp("\\b" + list[i] + "\\b", "gi"))); + if (matches.has("mark")) { + // 优先处理单词为 "mark" 的情况 + totalSet = new Set(["mark", ...totalSet]); } + totalSet = new Set([...totalSet, ...matches]); } + } + // 删除所有的mark标签,防止标签发生嵌套 + articleContent = articleContent.replace(/<(mark)[^>]*>/gi, ""); + articleContent = articleContent.replace(/<(\/mark)[^>]*>/gi, ""); + // 将文章中所有出现该单词word的地方改为："" + word + ""。 + for (let word of totalSet) { + articleContent = articleContent.replace(new RegExp("\\b" + word + "\\b", "g"), "" + word + ""); } document.getElementById("article").innerHTML = articleContent; } function cancelHighlighting() { - let articleContent = document.getElementById("article").innerText;//将原来的.innerText改为.innerHtml，原因同上 - let pickedWords = document.getElementById("selected-words"); - const dictionaryWords = document.getElementById("selected-words2"); - const list = pickedWords.value.split(" "); - if (pickedWords != null) { - for (let i = 0; i < list.length; ++i) { - list[i] = list[i].replace(/(^\s*)|(\s*$)/g, ""); - if (list[i] !== "") { //原来判断的代码中，替换的内容为“list[i]”这个字符串，这明显是错误的，我们需要替换的是list[i]里的内容 - articleContent = articleContent.replace(new RegExp(""+list[i]+"", "g"), list[i]); - } - } - } - if (dictionaryWords != null) { - let list2 = pickedWords.value.split(" "); - for (let i = 0; i < list2.length; ++i) { - list2 = dictionaryWords.value.split(" "); - list2[i] = list2[i].replace(/(^\s*)|(\s*$)/g, ""); - if (list2[i] !== "") { //原来代码中，替换的内容为“list[i]”这个字符串，这明显是错误的，我们需要替换的是list[i]里的内容 - articleContent = articleContent.replace(new RegExp(""+list2[i]+"", "g"), list2[i]); - } - } - } + let articleContent = document.getElementById("article").innerHTML; + articleContent = articleContent.replace(/<(mark)[^>]*>/gi, ""); + articleContent = articleContent.replace(/<(\/mark)[^>]*>/gi, ""); document.getElementById("article").innerHTML = articleContent; } @@ -99,4 +75,4 @@ function toggleHighlighting() { } } -showBtnHandler(); +showBtnHandler(); \ No newline at end of file diff --git a/app/static/js/read.js b/app/static/js/read.js index 814f627..c231930 100644 --- a/app/static/js/read.js +++ b/app/static/js/read.js @@ -3,13 +3,14 @@ var Reader = (function() { let current_position = 0; let original_position = 0; let to_speak = ""; + let current_rate = 1; // 添加这一行，设置默认速率为 1 function makeUtterance(str, rate) { let msg = new SpeechSynthesisUtterance(str); msg.rate = rate; msg.lang = "en-US"; msg.onboundary = ev => { - if (ev.name == "word") { + if (ev.name === "word") { current_position = ev.charIndex; } } @@ -24,12 +25,24 @@ var Reader = (function() { reader.speak(msg); } + function updateRate(rate) { + // 停止当前的朗读 + stopRead(); + + // 更新当前速率 + current_rate = rate; + + // 重新开始朗读 + read(to_speak, current_rate); + } + function stopRead() { reader.cancel(); } return { read: read, - stopRead: stopRead + stopRead: stopRead, + updateRate: updateRate // 添加这一行，将 updateRate 方法暴露出去 }; -})(); +}) (); diff --git a/app/static/js/word_operation.js b/app/static/js/word_operation.js index f043cce..dcf38ff 100644 --- a/app/static/js/word_operation.js +++ b/app/static/js/word_operation.js @@ -5,15 +5,14 @@ function familiar(theWord) { $.ajax({ type:"GET", url:"/" + username + "/" + word + "/familiar", - success:function(response){ + success:function(response) { let new_freq = freq - 1; const allow_move = document.getElementById("move_dynamiclly").checked; if (allow_move) { - if (new_freq <= 0) { removeWord(theWord); } else { - renderWord({ word: theWord, freq: new_freq }); + renderWord({word: theWord, freq: new_freq}); } } else { if(new_freq <1) { @@ -33,11 +32,11 @@ function unfamiliar(theWord) { $.ajax({ type:"GET", url:"/" + username + "/" + word + "/unfamiliar", - success:function(response){ + success:function(response) { let new_freq = parseInt(freq) + 1; const allow_move = document.getElementById("move_dynamiclly").checked; if (allow_move) { - renderWord({ word: theWord, freq: new_freq }); + renderWord({word: theWord, freq: new_freq}); } else { $("#freq_" + theWord).text(new_freq); } @@ -51,7 +50,7 @@ function delete_word(theWord) { $.ajax({ type:"GET", url:"/" + username + "/" + word + "/del", - success:function(response){ + success:function(response) { const allow_move = document.getElementById("move_dynamiclly").checked; if (allow_move) { removeWord(theWord); @@ -114,7 +113,7 @@ function removeWord(word) { // 根据词频信息删除元素 word = word.replace('&', '&'); const element_to_remove = document.getElementById(`p_${word}`); - if (element_to_remove != null) { + if (element_to_remove !== null) { element_to_remove.remove(); } } @@ -129,7 +128,7 @@ function renderWord(word) { for (const current of container.children) { const cur_word = parseWord(current); // 找到第一个词频比它小的元素，插入到这个元素前面 - if (compareWord(cur_word, word) == -1) { + if (compareWord(cur_word, word) === -1) { container.insertBefore(new_element, current); inserted = true; break; @@ -165,17 +164,11 @@ function elementFromString(string) { * 当first大于second时返回1 */ function compareWord(first, second) { - if (first.freq < second.freq) { - return -1; + if (first.freq !== second.freq) { + return first.freq < second.freq ? -1 : 1; } - if (first.freq > second.freq) { - return 1; - } - if (first.word < second.word) { - return -1; - } - if (first.word > second.word) { - return 1; + if (first.word !== second.word) { + return first.word < second.word ? -1 : 1; } return 0; -} +} \ No newline at end of file diff --git a/app/templates/login.html b/app/templates/login.html index 2507f75..b8eb118 100644 --- a/app/templates/login.html +++ b/app/templates/login.html @@ -1,7 +1,7 @@ {% block body %} {% if session['logged_in'] %} -你已登录 {{ session['username'] }}。 登出点击这里。 +你已登录 {{ session['username'] }}。 登出点击这里。 {% else %} @@ -32,14 +32,13 @@
-
{% endif %} diff --git a/app/templates/mainpage_get.html b/app/templates/mainpage_get.html index 344943d..e96e8dc 100644 --- a/app/templates/mainpage_get.html +++ b/app/templates/mainpage_get.html @@ -44,6 +44,7 @@ {{x[0]}} {{x[1]}} {% endfor %} {% endif %} +

Version: 20230810

{{ yml['footer'] | safe }} diff --git a/app/templates/signup.html b/app/templates/signup.html index 9030d41..5b3aafe 100644 --- a/app/templates/signup.html +++ b/app/templates/signup.html @@ -53,7 +53,7 @@ You're logged in already! Logout.

diff --git a/app/templates/userpage_get.html b/app/templates/userpage_get.html index f6ed768..4ab4001 100644 --- a/app/templates/userpage_get.html +++ b/app/templates/userpage_get.html @@ -23,44 +23,34 @@ EnglishPal Study Room for {{ username }} @@ -69,70 +59,59 @@

English Pal for {{ username }} {% if username == admin_name %} - 管理 + 管理 {% endif %} 退出 重设密码

{% for message in get_flashed_messages() %} - + {% endfor %} - - + +

阅读文章并回答问题

- -

Article added on: {{ today_article["date"] }}


+ +

Article added on: {{ today_article["date"] }}



-

{{ today_article["article_title"] }}


-

{{ today_article["article_body"] }}


-
-

{{ today_article['source'] }}


-
- -

{{ today_article['question'] }}


+

{{ today_article["article_title"] }}


+

{{ today_article["article_body"] }}


+
+

{{ today_article['source'] }}


+
+ +

{{ today_article['question'] }}


- -
+
- 生词高亮 - 大声朗读 - 划词入库 + 生词高亮 + 大声朗读 + 划词入库
@@ -145,7 +124,7 @@

- +
{% if session.get['thisWord'] %} diff --git a/app/templates/userpage_post.html b/app/templates/userpage_post.html index 1163787..a7ad569 100644 --- a/app/templates/userpage_post.html +++ b/app/templates/userpage_post.html @@ -1,45 +1,50 @@ - - - - + + + + - {{ yml['header'] | safe }} - {% if yml['css']['item'] %} + {{ yml['header'] | safe }} + {% if yml['css']['item'] %} {% for css in yml['css']['item'] %} {% endfor %} - {% endif %} - {% if yml['js']['head'] %} + {% endif %} + {% if yml['js']['head'] %} {% for js in yml['js']['head'] %} - + {% endfor %} - {% endif %} + {% endif %} - EnglishPal Study Room for {{username}} - - -

取消勾选认识的单词

-
- - {% for x in lst %} - {% set word = x[0]%} -

- {{loop.index}} - : - {{word}} - ({{x[1]}}) - -

+ EnglishPal Study Room for {{username}} + + +
+

+ + +

+ + + {% for x in lst %} + {% set word = x[0]%} +

+ {{loop.index}} + : + {{word}} + ({{x[1]}}) + +

- {% endfor %} - - {{ yml['footer'] | safe }} - {% if yml['js']['bottom'] %} - {% for js in yml['js']['bottom'] %} + {% endfor %} + + {{ yml['footer'] | safe }} + {% if yml['js']['bottom'] %} + {% for js in yml['js']['bottom'] %} - {% endfor %} - {% endif %} - + {% endfor %} + {% endif %} +
+ diff --git a/app/test/conftest.py b/app/test/conftest.py index af17368..ed4186c 100644 --- a/app/test/conftest.py +++ b/app/test/conftest.py @@ -1,6 +1,9 @@ import pytest +import sqlite3 +import time from selenium import webdriver -from selenium.webdriver.common.desired_capabilities import DesiredCapabilities + +from pathlib import Path @pytest.fixture def URL(): @@ -9,5 +12,24 @@ def URL(): @pytest.fixture def driver(): - my_driver = webdriver.Chrome() - return my_driver + return webdriver.Edge() # follow the "End-to-end testing" section in README.md to install the web driver executable + + +@pytest.fixture +def restore_sqlite_database(): + ''' + Automatically restore SQLite database file app/db/wordfreqapp.db + using SQL statements from app/static/wordfreqapp.sql + ''' + con = sqlite3.connect('../db/wordfreqapp.db') + with con: + con.executescript('DROP TABLE IF EXISTS user;') + con.executescript('DROP TABLE IF EXISTS article;') + con.executescript(open('../static/wordfreqapp.sql', encoding='utf8').read()) + con.close() + + +@pytest.fixture(autouse=True) +def restart_englishpal(restore_sqlite_database): + (Path(__file__).parent / '../main.py').touch() + time.sleep(1) diff --git a/app/test/helper.py b/app/test/helper.py new file mode 100644 index 0000000..2b0deec --- /dev/null +++ b/app/test/helper.py @@ -0,0 +1,33 @@ +import uuid +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.common.exceptions import UnexpectedAlertPresentException, NoAlertPresentException + +def signup(URL, driver): + username = 'TestUser' + str(uuid.uuid1()).split('-')[0].title() + password = '[Abc+123]' + + driver.get(URL) + + elem = driver.find_element_by_link_text('注册') + elem.click() + + elem = driver.find_element_by_id('username') + elem.send_keys(username) + + elem = driver.find_element_by_id('password') + elem.send_keys(password) + + elem = driver.find_element_by_id('password2') + elem.send_keys(password) + + elem = driver.find_element_by_class_name('btn') # 找到"注册"按钮 + elem.click() + + try: + WebDriverWait(driver, 1).until(EC.alert_is_present()) + driver.switch_to.alert.accept() + except (UnexpectedAlertPresentException, NoAlertPresentException): + pass + + return username, password diff --git a/app/test/test_add_word.py b/app/test/test_add_word.py index a08c376..6526bf0 100644 --- a/app/test/test_add_word.py +++ b/app/test/test_add_word.py @@ -1,76 +1,31 @@ -# -*- coding: utf-8 -*- -# Run the docker image using the following command: -# docker run -d -p 4444:4444 selenium/standalone-chrome -from selenium import webdriver -from selenium.webdriver.common.desired_capabilities import DesiredCapabilities - -import random, time -import string - -driver = webdriver.Remote('http://localhost:4444/wd/hub', DesiredCapabilities.FIREFOX) -driver.implicitly_wait(10) - -HOME_PAGE = 'http://121.4.94.30:91/' +import time +from helper import signup -def has_punctuation(s): - return [c for c in s if c in string.punctuation] != [] - -def test_add_word(): +def test_add_word(URL, driver): try: - driver.get(HOME_PAGE) - assert 'English Pal -' in driver.page_source - - # login - elem = driver.find_element_by_link_text('登录') - elem.click() - - uname = 'lanhui' - password = 'l0ve1t' - elem = driver.find_element_by_name('username') - elem.send_keys(uname) - - elem = driver.find_element_by_name('password') - elem.send_keys(password) - - elem = driver.find_element_by_xpath('//form[1]/p[3]/input[1]') # 找到登录按钮 - elem.click() - - assert 'EnglishPal Study Room for ' + uname in driver.title - - # get essay content - elem = driver.find_element_by_id('text-content') - essay_content = elem.text - - elem = driver.find_element_by_id('selected-words') - word = random.choice(essay_content.split()) - while 'font>' in word or 'br>' in word or 'p>' in word or len(word) < 6 or has_punctuation(word): - word = random.choice(essay_content.split()) + username, password = signup(URL, driver) # sign up a new account and automatically log in + time.sleep(1) + # enter the word in the text area + elem = driver.find_element_by_id('selected-words') + word = 'devour' elem.send_keys(word) - elem = driver.find_element_by_xpath('//form[1]//input[1]') # 找到get所有词频按钮 - elem.click() - - elems = driver.find_elements_by_xpath("//input[@type='checkbox']") - for elem in elems: - if elem.get_attribute('name') == 'marked': - elem.click() - - elem = driver.find_element_by_name('add-btn') # 找到加入我的生词簿按钮 + elem = driver.find_element_by_xpath('//form[1]//button[1]') # 找到"把生词加入我的生词库"按钮 + elem.click() + + elem = driver.find_element_by_name('add-btn') # 找到"加入我的生词簿"按钮 elem.click() - driver.refresh() - driver.refresh() - driver.refresh() elems = driver.find_elements_by_xpath("//p[@class='new-word']/a") - + found = 0 for elem in elems: if word in elem.text: found = 1 break - + assert found == 1 - finally: + finally: driver.quit() diff --git a/app/test/test_stress.py b/app/test/test_stress.py new file mode 100755 index 0000000..b437b86 --- /dev/null +++ b/app/test/test_stress.py @@ -0,0 +1,43 @@ +''' Contributed by Lin Junhong et al. 2023-06.''' + +import requests +import multiprocessing +import time + +def stress(username): + try: + data = { + 'username': username, + 'password': '123123' + } + headers = { + 'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36 Edg/114.0.1823.51' + } + session = requests.session() + response = session.post(url='http://127.0.0.1:5000/signup', data=data, headers=headers) + print('Sign up ', response.status_code) + time.sleep(0.5) + response = session.post(url='http://127.0.0.1:5000/login', data=data, headers=headers) + print('Sign in ', response.status_code) + time.sleep(0.5) + response = session.get(url=f'http://127.0.0.1:5000/{username}/userpage', headers=headers) + print('User page', response.status_code) + time.sleep(0.5) + print(session.cookies) + for i in range(5): + response = session.get(url=f'http://127.0.0.1:5000/get_next_article/{username}', headers=headers, cookies=session.cookies) + time.sleep(0.5) + print(f'Next page ({i}) [{username}]') + print(response.status_code) + print(response.json()['today_article']['article_title']) + except Exception as e: + print(e) + + +if __name__ == '__main__': + username = 'Learner' + pool = multiprocessing.Pool(processes=10) + for i in range(10): + pool.apply_async(stress, (f'{username}{i}',)) + pool.close() + pool.join() diff --git a/app/user_service.py b/app/user_service.py index 2e5feed..27323b8 100644 --- a/app/user_service.py +++ b/app/user_service.py @@ -15,6 +15,9 @@ from wordfreqCMD import sort_in_descending_order import pickle_idea import pickle_idea2 +import logging +logging.basicConfig(filename='log.txt', format='%(asctime)s %(message)s', level=logging.DEBUG) + # 初始化蓝图 userService = Blueprint("user_bp", __name__) @@ -32,7 +35,9 @@ def get_next_article(username): else: # 当前不为“null”，直接 index+=1 visited_articles["index"] += 1 session["visited_articles"] = visited_articles + logging.debug('/get_next_article: start calling get_today_arcile()') visited_articles, today_article, result_of_generate_article = get_today_article(user_freq_record, session.get('visited_articles')) + logging.debug('/get_next_arcile: done.') data = { 'visited_articles': visited_articles, 'today_article': today_article, @@ -129,7 +134,7 @@ def userpage(username): user_freq_record = path_prefix + 'static/frequency/' + 'frequency_%s.pickle' % (username) if request.method == 'POST': # when we submit a form - content = escape(request.form['content']) + content = request.form['content'] f = WordFreq(content) lst = f.get_freq() return render_template('userpage_post.html',username=username,lst = lst, yml=Yaml.yml) @@ -176,7 +181,11 @@ def user_mark_word(username): for word in request.form.getlist('marked'): lst.append((word, [get_time()])) d = pickle_idea2.merge_frequency(lst, lst_history) - pickle_idea2.save_frequency_to_pickle(d, user_freq_record) + if len(lst_history) > 999: + flash('You have way too many words in your difficult-words book. Delete some first.') + else: + pickle_idea2.save_frequency_to_pickle(d, user_freq_record) + flash('Added %s.' % (', '.join(request.form.getlist('marked')))) return redirect(url_for('user_bp.userpage', username=username)) else: return 'Under construction' diff --git a/build.sh b/build.sh index e313fce..158a86f 100755 --- a/build.sh +++ b/build.sh @@ -2,10 +2,7 @@ DEPLOYMENT_DIR=/home/lanhui/englishpal2/EnglishPal cd $DEPLOYMENT_DIR - -# Install dependencies - -pip3 install -r requirements.txt +pwd # Stop service sudo docker stop EnglishPal @@ -15,7 +12,7 @@ sudo docker rm EnglishPal sudo docker build -t englishpal . # Run the application -sudo docker run --restart=always -d --name EnglishPal -p 90:80 -v ${DEPLOYMENT_DIR}/app/static/frequency:/app/static/frequency -v ${DEPLOYMENT_DIR}/app/static/:/app/static/ -t englishpal # for permanently saving data +sudo docker run --restart=always -d --name EnglishPal -p 90:80 -v ${DEPLOYMENT_DIR}/app/static/frequency:/app/static/frequency --mount type=volume,src=englishpal-db,target=/app/db -t englishpal # for permanently saving data # Save space. Run it after sudo docker run sudo docker system prune -a -f