Use redis to transfer newly mapped data [not tested]

3 files changed, 34 insertions, 2 deletions

diff --git a/Code/configure.py b/Code/configure.py
index f12eed9..be04994 100644
--- a/Code/configure.py
+++ b/Code/configure.py
@@ -59,3 +59,6 @@ SQLITE_EDGE_FILE = '../Data/temp/edges.sqlite'
 DIFF_EDGE_FILE = '../Data/temp/edges-diff.txt' # the difference between two edge files from yesterday and from today
 
 TARGET_TF_FILE = '../Data/information/target_tf.txt'
+
+# Redis channel for sending newly mapped data
+REDIS_CHANNEL = 'SalmonQuant'
diff --git a/Code/download_and_map.py b/Code/download_and_map.py
index 502b5e5..bceeb82 100644
--- a/Code/download_and_map.py
+++ b/Code/download_and_map.py
@@ -19,11 +19,12 @@ import fnmatch
 import time
 import re
 import shutil
+import redis
+import json
 from datetime import datetime
 
 ##########################################################################################
-from configure import DAILY_MAP_NUMBER, MIN_FASTQ_FILE_SIZE, RNA_SEQ_INFO_FILE, DOWNLOADED_SRA_ID_LOG_FILE, IGNORED_SRA_ID_LOG_FILE, UPDATE_NETWORK_LOG_FILE, MAPPED_RDATA_DIR, RAW_RDATA_DIR, SALMON_MAP_RESULT_DIR
-
+from configure import DAILY_MAP_NUMBER, MIN_FASTQ_FILE_SIZE, RNA_SEQ_INFO_FILE, DOWNLOADED_SRA_ID_LOG_FILE, IGNORED_SRA_ID_LOG_FILE, UPDATE_NETWORK_LOG_FILE, MAPPED_RDATA_DIR, RAW_RDATA_DIR, SALMON_MAP_RESULT_DIR, REDIS_CHANNEL
 ##########################################################################################
 
 def glob_files(directory, pattern):
@@ -385,6 +386,16 @@ def reverse_lines(fname, fname2):
         content = ''.join(lines)
         f2.write(content)
 
+
+def publish(mapped_data_directory):
+    r = redis.Redis(host='118.25.96.118', port=6379, db=0)
+    for fname in glob.glob('%s/*_quant.txt' % (mapped_data_directory.rstrip('/'))):
+        file_basename = os.path.basename(fname)
+        with open(fname, 'r') as f:
+            data = f.read()
+        r.publish(REDIS_CHANNEL, json.dumps({'filename':file_basename, 'data':data}))
+        time.sleep(1)
+
 ## main
 
 # For filtering RNA-seq data
@@ -433,6 +444,7 @@ if not os.path.isdir(MAPPED_RDATA_DIR):
 
 # after mapping is finished, move all resulting files to MAPPED_RDATA_DIR
 if glob.glob('%s/*_quant.txt' % (SALMON_MAP_RESULT_DIR.rstrip('/'))) != []:
+    publish(SALMON_MAP_RESULT_DIR)
     cmd = 'mv %s/*_quant.txt %s' % (SALMON_MAP_RESULT_DIR.rstrip('/'), MAPPED_RDATA_DIR)
     os.system(cmd)
     print('[download_and_map.py] Done. Check directory %s.' % (os.path.abspath(MAPPED_RDATA_DIR)))
diff --git a/Code/get_mapped_data.py b/Code/get_mapped_data.py
new file mode 100644
index 0000000..d1a4146
--- /dev/null
+++ b/Code/get_mapped_data.py
@@ -0,0 +1,17 @@
+from configure import MAPPED_RDATA_DIR
+from configure import REDIS_CHANNEL
+from log import write_log_file
+import redis # install redis on Ubuntu using this command: sudo apt install python3-redis
+import json
+import os
+
+r = redis.Redis(host='118.25.96.118', port=6379, db=0)
+pubsub = r.pubsub(ignore_subscribe_messages=True)
+pubsub.subscribe(REDIS_CHANNEL)
+
+for message in pubsub.listen():
+    quant_data = json.loads(message['data'])
+    filename = os.path.join(MAPPED_RDATA_DIR, quant_data['filename'])
+    write_log_file('[get_mapped_data.py] Save %s.' % (filename), UPDATE_NETWORK_LOG_FILE))
+    with open(filename, 'w') as f:
+        f.write(quant_data['data'])