diff options
| -rw-r--r-- | Code/update_network_by_force.py | 20 |
1 files changed, 12 insertions, 8 deletions
diff --git a/Code/update_network_by_force.py b/Code/update_network_by_force.py index 55fd272..221be74 100644 --- a/Code/update_network_by_force.py +++ b/Code/update_network_by_force.py @@ -1,5 +1,5 @@ # Usage: python3 update_network_by_force.py -# Purpose: update_network.py could take a few days to run. Run this script to harvest new edges everyday. +# Purpose: update_network.py could take a few days (even weeks) to run. Run this script to harvest new edges everyday. # # Revision history: # Last modified: 24 Nov 2019, hui <lanhui@zjnu.edu.cn> @@ -106,7 +106,8 @@ cmd = 'curl http://118.25.96.118/brain/before' os.system(cmd) ########## Merge edges ####################### -# update edges.txt, a merged file from two sources, HISTORY_DIR and HISTORY_DIR2. Some new edge files are being generated ... +# Update edges.txt, a merged file from two sources, HISTORY_DIR and HISTORY_DIR2. Some new edge files are being generated ... +# Definition of HISTORY_DIR and HISTORY_DIR2 could be found in configure.py time.sleep(3) edge_file_lst = [] # collect edge files. most_recent_edge_modification_time = 0 @@ -117,7 +118,7 @@ for fname in glob.glob(os.path.join(HISTORY_DIR, 'edges.txt.*')): # many small e most_recent_edge_modification_time = os.path.getmtime(fname) write_log_file('[update_network_by_force.py] Look at edge files in %s.' % (HISTORY_DIR2), UPDATE_NETWORK_LOG_FILE) -for fname in glob.glob(os.path.join(HISTORY_DIR2, 'edges.txt.*')): # edges.txt.* are to be merged +for fname in glob.glob(os.path.join(HISTORY_DIR2, 'edges.txt.*')): # many edges.txt.* are to be merged edge_file_lst.append(fname) if os.path.getmtime(fname) > most_recent_edge_modification_time: most_recent_edge_modification_time = os.path.getmtime(fname) @@ -130,19 +131,22 @@ elif os.path.getmtime(MERGED_EDGE_FILE) < most_recent_edge_modification_time: # write_log_file('[update_network_by_force.py] Concatenate edge files in %s and %s into one file.' % (HISTORY_DIR, HISTORY_DIR2), UPDATE_NETWORK_LOG_FILE) curr_time = datetime.now().strftime('%Y%m%d_%H%M') concatenate_edge_files(edge_file_lst, os.path.join(EDGE_POOL_DIR, 'edges.txt.many.one.targets.' + curr_time)) - delete_edge_files(edge_file_lst) + delete_edge_files(edge_file_lst) # delete these files only when they are no longer being written. -if os.path.getmtime(MERGED_EDGE_FILE) < os.path.getmtime(EDGE_POOL_DIR): # edge pool directory has been updated, create new edges.txt +if os.path.getmtime(MERGED_EDGE_FILE) < os.path.getmtime(EDGE_POOL_DIR): # edge pool directory has been updated, create a new edges.txt write_log_file('[update_network_by_force.py] Make a new edges.txt from edge files in %s.' % (EDGE_POOL_DIR), UPDATE_NETWORK_LOG_FILE) write_log_file('[update_network_by_force.py] Number of lines in the old edges.txt: %d.' % (num_line(MERGED_EDGE_FILE)), UPDATE_NETWORK_LOG_FILE) write_log_file('[update_network_by_force.py] %s' % (summarize_edge_file(MERGED_EDGE_FILE)), UPDATE_NETWORK_LOG_FILE) - cmd = 'python3 merge_edges.py' - os.system(cmd) + cmd = 'python3 merge_edges.py' # invoke another script the merge all edge files in EDGE_POOL_DIR + return_value = os.system(cmd) + if return_value != 0: + write_log_file('[update_network_by_force.py] Something wrong occurred to merge_edges.py. Perhaps your computer is running out of memory.', UPDATE_NETWORK_LOG_FILE) write_log_file('[update_network_by_force.py] Number of lines in the new edges.txt: %d.' % (num_line(MERGED_EDGE_FILE)), UPDATE_NETWORK_LOG_FILE) write_log_file('[update_network_by_force.py] %s' % (summarize_edge_file(MERGED_EDGE_FILE)), UPDATE_NETWORK_LOG_FILE) manual_copy_commands = 'Please copy files to the web application: sudo cp /home/lanhui/brain/Data/temp/edges.txt /var/www/brain/brain/static/edges/edges.txt sudo cp /home/lanhui/brain/Data/temp/html_edges/edges.sqlite /var/www/brain/brain/static/edges curl http://118.25.96.118/brain/before' write_log_file('[update_network_by_force.py] %s' % (manual_copy_commands), UPDATE_NETWORK_LOG_FILE) - cmd = 'html_network.py -f %s -r %s -c %s -n %s' % (MERGED_EDGE_FILE, PARAMETER_FOR_BUILDRMATRIX, PARAMETER_FOR_BUILDCMATRIX, PARAMETER_FOR_NET) + write_log_file('[update_network_by_force.py] Make html files for the web application.', UPDATE_NETWORK_LOG_FILE) + cmd = 'python3 html_network.py -f %s -r %s -c %s -n %s' % (MERGED_EDGE_FILE, PARAMETER_FOR_BUILDRMATRIX, PARAMETER_FOR_BUILDCMATRIX, PARAMETER_FOR_NET) os.system(cmd) copy_and_backup_file(MERGED_EDGE_FILE, '../Analysis') # the backup file will be used for further analysis |