Merge pull request #169 from icgc-argo-workflows/payload-gen-seq-experiment@0.8.3

[release]

Author: edsu7

payload-gen-seq-experiment/main.nf

@@ -26,7 +26,7 @@
 /* this block is auto-generated based on info from pkg.json where   */
 /* changes can be made if needed, do NOT modify this block manually */
 nextflow.enable.dsl = 2
-version = '0.8.2'
+version = '0.8.3'
 
 container = [
     'ghcr.io': 'ghcr.io/icgc-argo-workflows/data-processing-utility-tools.payload-gen-seq-experiment'

payload-gen-seq-experiment/main.py

@@ -45,10 +45,12 @@
     'submitter_sample_id','sample_type', 'submitter_matched_normal_sample_id', 'sequencing_center', 
     'platform', 'platform_model','experimental_strategy', 'sequencing_date', 'read_group_count']
 TSV_FIELDS['experiment']["conditional"]=[
-    "library_isolation_protocol","library_preparation_kit",
-    "library_strandedness","rin","dv200","spike_ins_included",
-    "spike_ins_fasta","spike_ins_concentration",
-    "target_capture_kit"]
+    "library_preparation_kit",
+    "library_strandedness",
+    "rin","dv200",
+    "target_capture_kit","number_of_genes","gene_padding","coverage",
+    "primary_target_regions","capture_target_regions"
+    ]
 
 TSV_FIELDS['read_group']= {}
 TSV_FIELDS['read_group']["core"]=[
@@ -77,10 +79,10 @@
 
 def empty_str_to_null(metadata):
     for k in metadata:
-        if k in ['read_groups', 'files']:
+        if k in ['read_groups', 'files','experiment']:
             for i in range(len(metadata[k])):
                 empty_str_to_null(metadata[k][i])
-        if isinstance(metadata[k], str) and metadata[k] in ["", "_NULL_"]:
+        if isinstance(metadata[k], str) and metadata[k] in ["", "_NULL_","null","NULL","Null","None","NONE","none"]:
             metadata[k] = None
 
 
@@ -146,10 +148,20 @@ def load_all_tsvs(exp_tsv, rg_tsv, file_tsv):
                 rg['is_paired_end'] = None
 
             for field in ('read_length_r1', 'read_length_r2', 'insert_size'):
-                if rg[field]:
+                if isinstance(rg[field],str):
+                    if re.match("^[0-9]+$", rg[field]):
+                        rg[field] = int(rg[field])
+                        continue
+                    for empty_string in ["", "_NULL_",'null',"NULL","Null","None","NONE","none"]:
+                        if rg[field]==empty_string:
+                            rg[field] = None
+                            break
+                elif isinstance(rg[field],int):
                     rg[field] = int(rg[field])
-                else:
+                elif rg[field] is None:
                     rg[field] = None
+                else:
+                    sys.exit("Unrecognnized value '%s' in field %s for '%s'" % (str(rg[field]),field,rg['submitter_read_group_id']))
 
             metadata_dict['read_groups'].append(rg)
 
@@ -262,7 +274,7 @@ def main(metadata,url,bam_from_cram,bam_from_cram_reference,recalculate_size_and
     optional_experimental_fields.remove("rin")
 
     for optional_experimental_field in optional_experimental_fields:
-        if metadata.get(optional_experimental_field):
+        if optional_experimental_field in metadata.keys():
             payload['experiment'][optional_experimental_field]=metadata.get(optional_experimental_field)
     # Int
     optional_experimental_fields=["rin"]
@@ -274,6 +286,12 @@ def main(metadata,url,bam_from_cram,bam_from_cram_reference,recalculate_size_and
     if metadata.get('experimental_strategy')=='RNA-Seq' and not metadata.get("library_strandedness"):
         sys.exit(f"'experimental_strategy' 'RNA-Seq' specified but 'library_strandedness' is missing. Resubmit with both values 'experimental_strategy' and 'library_strandedness'")
 
+    # Targetted Sequencing :
+    if metadata.get('experimental_strategy')=="Targeted-Seq" or metadata.get('experimental_strategy')=="WXS":
+        for field in ['target_capture_kit','primary_target_regions','capture_target_regions']:
+            if field not in metadata.keys():
+                sys.exit(f"'experimental_strategy' '%s' specified but '%s' is missing. Resubmit with both values 'experimental_strategy' and '%s'" % (metadata.get('experimental_strategy'),field,field))
+
     # get sample of the payload
     sample = {
         'submitterSampleId': metadata.get('submitter_sample_id'),
@@ -449,4 +467,4 @@ def main(metadata,url,bam_from_cram,bam_from_cram_reference,recalculate_size_and
                     extra_info[row_type][row_id][row_field]=row_val
 
 
-        main(metadata,url,args.bam_from_cram,args.bam_from_cram_reference,args.recalculate_size_and_md5_files,extra_info)
+        main(metadata,url,args.bam_from_cram,args.bam_from_cram_reference,args.recalculate_size_and_md5_files,extra_info)

payload-gen-seq-experiment/pkg.json

@@ -1,6 +1,6 @@
 {
     "name": "payload-gen-seq-experiment",
-    "version": "0.8.2",
+    "version": "0.8.3",
     "description": "SONG payload generation for sequencing experiment",
     "main": "main.nf",
     "deprecated": false,

payload-gen-seq-experiment/tests/checker.nf

@@ -31,7 +31,7 @@
 /* this block is auto-generated based on info from pkg.json where   */
 /* changes can be made if needed, do NOT modify this block manually */
 nextflow.enable.dsl = 2
-version = '0.8.2'
+version = '0.8.3'
 
 container = [
     'ghcr.io': 'ghcr.io/icgc-argo-workflows/data-processing-utility-tools.payload-gen-seq-experiment'

payload-gen-seq-experiment/tests/expected/b9167a75-83ea-4c43-be30-e87faf3557dd.sequencing_experiment.payload.json

@@ -0,0 +1,89 @@
+{
+  "analysisType": {
+    "name": "sequencing_experiment"
+  },
+  "studyId": "TEST-PRO",
+  "experiment": {
+    "submitter_sequencing_experiment_id": "TEST_EXP",
+    "sequencing_center": "EXT",
+    "platform": "ILLUMINA",
+    "platform_model": "HiSeq 2000",
+    "experimental_strategy": "WXS",
+    "sequencing_date": "2014-12-12",
+    "target_capture_kit": "DUMMY_VAL",
+    "primary_target_regions": "DUMMY_VAL",
+    "capture_target_regions": "DUMMY_VAL"
+  },
+  "read_group_count": 3,
+  "read_groups": [
+    {
+      "submitter_read_group_id": "C0HVY.2",
+      "read_group_id_in_bam": null,
+      "platform_unit": "74_8a",
+      "is_paired_end": true,
+      "file_r1": "test_rg_3.bam",
+      "file_r2": "test_rg_3.bam",
+      "read_length_r1": null,
+      "read_length_r2": null,
+      "insert_size": 298,
+      "sample_barcode": null,
+      "library_name": "Pond-147580"
+    },
+    {
+      "submitter_read_group_id": "D0RE2.1",
+      "read_group_id_in_bam": null,
+      "platform_unit": "74_8b",
+      "is_paired_end": true,
+      "file_r1": "test_rg_3.bam",
+      "file_r2": "test_rg_3.bam",
+      "read_length_r1": null,
+      "read_length_r2": null,
+      "insert_size": 298,
+      "sample_barcode": null,
+      "library_name": "Pond-147580"
+    },
+    {
+      "submitter_read_group_id": "D0RH0.2",
+      "read_group_id_in_bam": null,
+      "platform_unit": "74_8c",
+      "is_paired_end": true,
+      "file_r1": "test_rg_3.bam",
+      "file_r2": "test_rg_3.bam",
+      "read_length_r1": null,
+      "read_length_r2": null,
+      "insert_size": 298,
+      "sample_barcode": null,
+      "library_name": "Pond-147580"
+    }
+  ],
+  "samples": [
+    {
+      "submitterSampleId": "HCC1143_BAM_INPUT",
+      "matchedNormalSubmitterSampleId": null,
+      "sampleType": "Total DNA",
+      "specimen": {
+        "submitterSpecimenId": "HCC1143_BAM_INPUT",
+        "tumourNormalDesignation": "Normal",
+        "specimenTissueSource": "Blood derived",
+        "specimenType": "Cell line - derived from normal"
+      },
+      "donor": {
+        "submitterDonorId": "HCC1143",
+        "gender": "Female"
+      }
+    }
+  ],
+  "files": [
+    {
+      "fileName": "test_rg_3.bam",
+      "fileSize": 14911,
+      "fileMd5sum": "178f97f7b1ca8bfc28fd5586bdd56799",
+      "fileType": "BAM",
+      "fileAccess": "controlled",
+      "dataType": "Submitted Reads",
+      "info": {
+        "data_category": "Sequencing Reads"
+      }
+    }
+  ]
+}

payload-gen-seq-experiment/tests/input/experiment.WXS.tsv

@@ -0,0 +1,2 @@
+type	program_id	submitter_sequencing_experiment_id	submitter_donor_id	gender	submitter_specimen_id	tumour_normal_designation	specimen_type	specimen_tissue_source	submitter_sample_id	sample_type	submitter_matched_normal_sample_id	sequencing_center	platform	platform_model	experimental_strategy	sequencing_date	read_group_count	target_capture_kit	primary_target_regions	capture_target_regions
+sequencing_experiment	TEST-PRO	TEST_EXP	HCC1143	Female	HCC1143_BAM_INPUT	Normal	Cell line - derived from normal	Blood derived	HCC1143_BAM_INPUT	Total DNA		EXT	ILLUMINA	HiSeq 2000	WXS	2014-12-12	3	DUMMY_VAL	DUMMY_VAL	DUMMY_VAL

payload-gen-seq-experiment/tests/input/file.WXS.tsv

@@ -0,0 +1,2 @@
+type	name	format	size	md5sum	path
+file	test_rg_3.bam	BAM	14911	178f97f7b1ca8bfc28fd5586bdd56799	test_rg_3.bam

payload-gen-seq-experiment/tests/input/read_group.WXS.tsv

@@ -0,0 +1,4 @@
+type	submitter_read_group_id	read_group_id_in_bam	submitter_sequencing_experiment_id	platform_unit	is_paired_end	file_r1	file_r2	read_length_r1	read_length_r2	insert_size	sample_barcode	library_name
+read_group	C0HVY.2		TEST_EXP	74_8a	true	test_rg_3.bam	test_rg_3.bam	null		298		Pond-147580
+read_group	D0RE2.1		TEST_EXP	74_8b	true	test_rg_3.bam	test_rg_3.bam	NULL	Null	298		Pond-147580
+read_group	D0RH0.2		TEST_EXP	74_8c	true	test_rg_3.bam	test_rg_3.bam	NONE	None	298		Pond-147580

payload-gen-seq-experiment/tests/test-job-bam.wxs.json

@@ -0,0 +1,7 @@
+{
+  "experiment_info_tsv": "input/experiment.WXS.tsv",
+  "read_group_info_tsv": "input/read_group.WXS.tsv",
+  "file_info_tsv": "input/file.WXS.tsv",
+  "expected_output": "expected/b9167a75-83ea-4c43-be30-e87faf3557dd.sequencing_experiment.payload.json",
+  "publish_dir": "outdir"
+}