From 6701b310b0f2694b681517381ad889ec67012fe0 Mon Sep 17 00:00:00 2001 From: Hui Lan Date: Tue, 21 Jan 2020 11:52:09 +0800 Subject: merge_edges.py: write all edge information to an SQLite database file called edges.sqlite When I saved a static html page for each edge (e.g., http://118.25.96.118/static/edges/AT1G20910_AT1G30100_0.html), it took 5GB disk space for saving 1 million html pages. Not very disk space efficient. An alternative is to save all edge information in a database table (i.e., edge), and query this database table for a particular edge. The database file edges.sqlite takes less than 200MB for 1 million edges, requiring 10 times smaller space than the static approach. The reason is that we do not have a lot of HTML tags in the database. Quite happy about that, though it seems that filling a database is a bit slower (2 hours??? for 1 million rows). Also updated two files that were affected: update_network.py and update_network_by_force.py. Now instead of copying 1 million static html page to the Webapp, I just need to copy edges.sqlite to static/edges/. Faster. In the Webapp, I updated start_webapp.py and added a file templates/edge.html for handling dynamic page generation. -Hui --- Code/update_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Code/update_network.py') diff --git a/Code/update_network.py b/Code/update_network.py index f65b3ba..672dc1e 100755 --- a/Code/update_network.py +++ b/Code/update_network.py @@ -865,7 +865,7 @@ if os.path.getmtime(MERGED_EDGE_FILE) < os.path.getmtime(EDGE_POOL_DIR): # edge cmd = 'python3 merge_edges.py' os.system(cmd) write_log_file('[update_network.py] Number of lines in the new edges.txt: %d.' % (num_line(MERGED_EDGE_FILE)), UPDATE_NETWORK_LOG_FILE) - manual_copy_commands = 'Please copy files to the web application: sudo cp /home/lanhui/brain/Data/temp/edges.txt /var/www/brain/brain/static/edges/edges.txt sudo find /home/lanhui/brain/Data/temp/html_edges -name "*.html" -exec mv -t /var/www/brain/brain/static/edges {} +' + manual_copy_commands = 'Please copy files to the web application: sudo cp /home/lanhui/brain/Data/temp/edges.txt /var/www/brain/brain/static/edges/edges.txt sudo cp /home/lanhui/brain/Data/temp/html_edges/edges.sqlite /var/www/brain/brain/static/edges' write_log_file('[update_network.py] %s' % (manual_copy_commands), UPDATE_NETWORK_LOG_FILE) -- cgit v1.2.1