Bug fixes

.gitignore

@@ -1,6 +1,6 @@
 # large files
 snapshot
-LOGS
+LOGS*
 LOGS_Tagging
 *.bin
 secrets

config.py

@@ -19,6 +19,7 @@ class Config(object):
     PICKLE_DIR = os.environ['PICKLE_DIR']
     SNAPSHOT_DIR = os.environ['SNAPSHOT_DIR']
     CUDA = os.environ['CUDA']
+    print('CUDA', CUDA, CUDA=='true')
 
     DB_TRAIN_PATH = os.path.join(PICKLE_DIR, "reportDBAPI_train.p")
     DB_BASE_PATH = os.path.join(PICKLE_DIR, "reportDB_base_train.p")
@@ -49,7 +50,7 @@ class Config(object):
     PRUNE_KEY = "OrganBreast"
     PRUNE_AFTER_PREDICT = False
 
-    COLUMN_KEYS = parsing.parse_XLS( os.environ['CONFIG_XLSX'])
+    COLUMN_KEYS = parsing.parse_XLS(os.environ['CONFIG_XLSX'])
 
     DIAGNOSES = {o: {} for o in COLUMN_KEYS.keys()}
     for organ in DIAGNOSES:
@@ -61,6 +62,7 @@ class Config(object):
     post_diagnoses['OrganBreast']['ER_Intensity'] = ['0', '1', '2', '3', '9']
     post_diagnoses['OrganBreast']['PR_Intensity'] = ['0', '1', '2', '3', '9']
     POST_DIAGNOSES = post_diagnoses
+    ORGANS = post_diagnoses.keys()
 
     CANCERS = ['ILC', 'DCIS', 'IDC', 'TubularCancer', 'CancerInvasive', 'CancerInvNOS', 'CancerNotOfBreastOrigin']
 
@@ -69,7 +71,7 @@ class Config(object):
     MARKERS = ['ER', "ER_Intensity", 'PR', "PR_Intensity", "her2", 'Her2Fish', "Her2_IHC", 'PositiveLN', 'ECE', 'ITC', 'BVI', 'LVI']
 
     RATIONALE_NET_CONFIG = {
-        'cuda': CUDA,
+        'cuda': True,
         'num_workers': 8,
         'train_batch_size': 32,
         'pred_batch_size': 200,

oncotext/evaluation.py

@@ -10,8 +10,7 @@
 import oncotext.utils.generic as generic
 import sklearn.metrics
 
-def score_on_test_set(reports, test_set, config, logger):
-    organ = generic.getOrgan(test_set[0], config)
+def score_on_test_set(reports, test_set, organ, config, logger):
     gold_reports = preprocess.apply_rules(
         test_set,
         organ,
@@ -83,15 +82,15 @@ def score_on_test_set(reports, test_set, config, logger):
     return results, keys
 
 
-def evaluate(reportDB, eval_sets, config, logger):
+def evaluate(reportDB, eval_sets, organ, config, logger):
     all_results = {}
 
     for file_name in eval_sets:
         test_set = eval_sets[file_name]
         relevant_reports = [r for r in reportDB if r['filename'] == file_name]
         logger.info("Scoring reportDB against test_set {}".format(file_name))
         logger.info("Scoring reportDB has {} records matching  test_set {} records".format(len(relevant_reports), len(test_set)))
-        results, result_keys = score_on_test_set(relevant_reports, test_set, config, logger)
+        results, result_keys = score_on_test_set(relevant_reports, test_set, organ, config, logger)
         all_results[file_name]= results
 
     return all_results

oncotext/utils/generic.py

@@ -22,8 +22,8 @@ def hasCat(r, cat, loose=False):
     return False
 
 
-def contains_annotations(reports, config):
-    diagnoses = config['DIAGNOSES']
+def contains_annotations(reports, organ, config):
+    diagnoses = config['DIAGNOSES'][organ]
     for r in reports:
         if hasCat(r, diagnoses, loose=True):
             return True

oncotext/utils/parsing.py

@@ -13,7 +13,7 @@ def parse_XLS(path):
             values = [str(cell.value) for cell in row if cell.value is not None]
             if len(values) > 1:
                 data[sheet.title][values[0]] = values[1: ]
-            elif len(values) == 0:
+            elif len(values) == 1:
                 data[sheet.title][values[0]] = []
     return data
 

oncotext/utils/postprocess.py

@@ -111,10 +111,23 @@ def generate_automatic_feilds(reportDB, organ, config):
             if 'CancerInvasive' in r and r['CancerInvasive'] == '0':
                 r['GradeMaxInvasive'] = '9'
 
-    elif organ == "OrganProstate":
+    elif organ == "Meta":
+        for r in reportDB:
+            if r['OrganProstate'] == '1':
+                if r['BiopsyType'] == 'Core':
+                    r['OrganProstateCore'] = '1'
+                    r['OrganProstateNonCore'] = '0'
+                else:
+                    r['OrganProstateCore'] = '0'
+                    r['OrganProstateNonCore'] = '1'
+            else:
+                r['OrganProstateCore'] = '0'
+                r['OrganProstateNonCore'] = '0'
+
+    elif organ == "OrganProstateCore" or organ == "OrganProstateNonCore":
         for r in reportDB:
             if r['ProstateCa'] == '0':
-                numerical = [k for k in config['POST_DIAGNOSES']['OrganProstate'] if config['POST_DIAGNOSES']['OrganProstate'][k] == ["NUM"]]
+                numerical = [k for k in config['POST_DIAGNOSES']['OrganProstateCore'] if config['POST_DIAGNOSES']['OrganProstateCore'][k] == ["NUM"]]
                 for k in numerical:
                     r[k] = '0'
 

oncotext/utils/preprocess.py

@@ -149,7 +149,8 @@ def segment_prostate(report, raw_text_key, preprocessed_text_key, segment_id_key
                     segments[0][0] += line+"\n"
                     segments[0][1] = alpha[0]+"."
                 else:
-                    segments.append([line+"\n", alpha[len(segments)]+"."])
+                    segment_name = alpha[len(segments)%len(alpha)]
+                    segments.append([line+"\n", segment_name + "."])
             else:
                 segments[-1][0] += line+"\n"
 
@@ -171,7 +172,7 @@ def segment_prostate(report, raw_text_key, preprocessed_text_key, segment_id_key
 def segment_reports(r, organ, raw_text_key, preprocessed_text_key, side_key, segment_id_key, segment_type_key, logger):
     if organ == "OrganBreast":
         segmented_reports = segment_breast(r, raw_text_key, preprocessed_text_key, side_key, logger)
-    elif organ == "OrganProstate":
+    elif organ == "OrganProstateCore":
         segmented_reports = segment_prostate(r, raw_text_key, preprocessed_text_key, segment_id_key, segment_type_key, logger)
     else:
         r[preprocessed_text_key] = preprocess_text(r[raw_text_key])
@@ -231,6 +232,9 @@ def apply_rules(reports, organ, raw_text_key, preprocessed_text_key, time_key, s
         r[raw_text_key] = r[raw_text_key] if raw_text_key in r else r[preprocessed_text_key]
         r[raw_text_key] = remove_bad_chars(r[raw_text_key])
 
+        if organ == 'Meta':
+            r[preprocessed_text_key] = r[raw_text_key]
+
         if preprocessed_text_key in r:
             r[preprocessed_text_key] = preprocess_text(r[preprocessed_text_key])
             preprocessed_reports.append(r)

scripts/app.py

@@ -66,7 +66,7 @@ def addTrainData():
                                   config['SEGMENT_TYPE_KEY'],
                                   logger)
 
-    if len(data) == 0 or not generic.contains_annotations(data, config):
+    if len(data) == 0 or not generic.contains_annotations(data, organ, config):
         logger.warn("addTrain[ - did not include any reports with labels. No op.")
         return NOP_MSG
 
@@ -187,7 +187,7 @@ def predict():
         eval_sets = json.loads(request.data.decode())
     except Exception as e:
         eval_sets = {}
-        logger.warn("No eval sets provided for prediction!")
+        logger.warn("No eval sets provided for prediction!", e)
 
     filename = DB_UNLABLED_PATH+"_"+organ+".p"
     db_unlabeled = pickle.load(open(filename, 'rb'))
@@ -216,7 +216,7 @@ def predict():
                                        config,
                                        logger)
 
-    results = evaluation.evaluate(reportDB, eval_sets, config, logger)
+    results = evaluation.evaluate(reportDB, eval_sets, organ, config, logger)
 
     return json.dumps({'reportDB': json_utils.make_json_compliant(reportDB),
                        'results': results,