diff options
Diffstat (limited to 'Code/update_network.py')
| -rwxr-xr-x | Code/update_network.py | 74 |
1 files changed, 0 insertions, 74 deletions
diff --git a/Code/update_network.py b/Code/update_network.py index d5aed9b..7b26f58 100755 --- a/Code/update_network.py +++ b/Code/update_network.py @@ -436,7 +436,6 @@ def need_update_parameter_file(param_file, dirs): return result - def validate_binding_file(fname): f = open(fname) lines = f.readlines() @@ -448,40 +447,6 @@ def validate_binding_file(fname): return True -def lines_with_10_fields(s): - result = [] - for line in s.split('\n'): - line = line.strip() - if len(line.split('\t')) == 10: - result.append(line) - return result - - -def concatenate_edge_files(fname_lst, fname_out): - fout = open(fname_out, 'w') - for fname in fname_lst: - f = open(fname) - s = f.read() - f.close() - # Make sure each edge has 10 fields before writing. - lines = lines_with_10_fields(s) - if lines != []: - write_log_file('[update_network.py] In function concatenate_edge_files. File %s has %d rows with 10 columns.' % (fname, len(lines)), UPDATE_NETWORK_LOG_FILE) - fout.write('\n'.join(lines) + '\n') - else: - write_log_file('[update_network.py] In function concatenate_edge_files. Check file %s. It has no rows with 10 fields.' % (fname), UPDATE_NETWORK_LOG_FILE) - fout.close() - - -def delete_edge_files(fname_lst): - for fname in fname_lst: - # Before we delete, we should make sure it is not being written. Make sure it is old enough. Otherwise, don't delete. - if age_of_file_in_seconds(fname) > 12*60*60: # 10 minutes - os.remove(fname) - else: - write_log_file('[update_network.py] In function delete_edge_files. Check file %s. It is probably still being written. So I don\'t delete it.' % (fname), UPDATE_NETWORK_LOG_FILE) - - def create_edges0(): if os.path.exists(PARAMETER_FOR_NET): write_log_file('[update_network.py] Create simple edges.txt using create_edges0.py with %s' % (PARAMETER_FOR_NET), UPDATE_NETWORK_LOG_FILE) @@ -765,49 +730,11 @@ if 'TPM.txt' in updated_file_list: # we could _touch_ TPM.txt to make it recent. #correlation_mixtools(3) -########## Merge edges ####################### -# update edges.txt, a merged file from two sources, HISTORY_DIR and HISTORY_DIR2. Some new edge files are being generated ... -time.sleep(5) -edge_file_lst = [] # collect edge files. -most_recent_edge_modification_time = 0 -write_log_file('[update_network.py] Look at edge files in %s.' % (HISTORY_DIR), UPDATE_NETWORK_LOG_FILE) -for fname in glob.glob(os.path.join(HISTORY_DIR, 'edges.txt.*')): # many small edges.txt.* are to be merged - edge_file_lst.append(fname) - if os.path.getmtime(fname) > most_recent_edge_modification_time: - most_recent_edge_modification_time = os.path.getmtime(fname) - -write_log_file('[update_network.py] Look at edge files in %s.' % (HISTORY_DIR2), UPDATE_NETWORK_LOG_FILE) -for fname in glob.glob(os.path.join(HISTORY_DIR2, 'edges.txt.*')): # edges.txt.* are to be merged - edge_file_lst.append(fname) - if os.path.getmtime(fname) > most_recent_edge_modification_time: - most_recent_edge_modification_time = os.path.getmtime(fname) - - -if edge_file_lst == []: - write_log_file('[update_network.py] No edge files to merge in %s and %s.' % (HISTORY_DIR, HISTORY_DIR2), UPDATE_NETWORK_LOG_FILE) -elif os.path.getmtime(MERGED_EDGE_FILE) < most_recent_edge_modification_time: # update edges.txt only if there are newer edges to add. - # concatenate edge files into one - write_log_file('[update_network.py] Concatenate edge files in %s and %s into one file.' % (HISTORY_DIR, HISTORY_DIR2), UPDATE_NETWORK_LOG_FILE) - curr_time = datetime.now().strftime('%Y%m%d_%H%M') - concatenate_edge_files(edge_file_lst, os.path.join(EDGE_POOL_DIR, 'edges.txt.many.one.targets.' + curr_time)) - delete_edge_files(edge_file_lst) - -if os.path.getmtime(MERGED_EDGE_FILE) < os.path.getmtime(EDGE_POOL_DIR): # edge pool directory has been updated, create new edges.txt - write_log_file('[update_network.py] Make a new edges.txt from edge files in %s.' % (EDGE_POOL_DIR), UPDATE_NETWORK_LOG_FILE) - write_log_file('[update_network.py] Number of lines in the old edges.txt: %d.' % (num_line(MERGED_EDGE_FILE)), UPDATE_NETWORK_LOG_FILE) - cmd = 'python3 merge_edges.py' - os.system(cmd) - write_log_file('[update_network.py] Number of lines in the new edges.txt: %d.' % (num_line(MERGED_EDGE_FILE)), UPDATE_NETWORK_LOG_FILE) - manual_copy_commands = 'Please copy files to the web application: sudo cp /home/lanhui/brain/Data/temp/edges.txt /var/www/brain/brain/static/edges/edges.txt sudo cp /home/lanhui/brain/Data/temp/html_edges/edges.sqlite /var/www/brain/brain/static/edges' - write_log_file('[update_network.py] %s' % (manual_copy_commands), UPDATE_NETWORK_LOG_FILE) - - # exclude edges as suggested by Phil Wigge. # write_log_file('Exclude edges (now ineffective)', UPDATE_NETWORK_LOG_FILE) # cmd = 'python3 exclude_edges.py %s' % (EDGE_FILE) #os.system(cmd) - # # check if parameter_for_net.txt, or TPM.txt is updated, if yes, create edges. # updated_file_list = get_updated_files(FILE_LIST_TO_CHECK, timestamp_dict) # if ('parameter_for_net.txt' in updated_file_list or 'TPM.txt' in updated_file_list) and not hold_on(PARAMETER_FOR_NET): @@ -826,4 +753,3 @@ os.system(cmd) record_file_time(FILE_LIST_TO_CHECK, FILE_TIMESTAMP) write_log_file('[update_network.py] Update done at %s.\n\n' % (datetime.now().strftime('%Y-%m-%d %H:%M:%S')), UPDATE_NETWORK_LOG_FILE) - |