Merge pull request #1186 from fls-bioinformatics-core/make_fastqs-no-undetermined

Add '--no-undetermined-fastqs' option to the 'make_fastqs' command

Author: pjbriggs

auto_process_ngs/bcl2fastq/pipeline.py

@@ -124,6 +124,7 @@
     'i2_length',
     'override_template',
     'no_lane_splitting',
+    'no_undetermined_fastqs',
     'tenx_filter_single_index',
     'tenx_filter_dual_index',
     'spaceranger_rc_i2_override',
@@ -355,6 +356,7 @@ def __init__(self,run_dir,sample_sheet,protocol='standard',
         self.add_param('find_adapters_with_sliding_window',value=False,
                        type=bool)
         self.add_param('create_empty_fastqs',value=False,type=bool)
+        self.add_param('no_undetermined_fastqs',value=False,type=bool)
         self.add_param('ignore_missing_bcls',value=False,type=bool)
         self.add_param('name',type=str)
         self.add_param('stats_file',type=str)
@@ -484,6 +486,7 @@ def __init__(self,run_dir,sample_sheet,protocol='standard',
                 adapter_sequence=self._adapter_sequence,
                 adapter_sequence2=self._adapter_sequence_read2,
                 no_lane_splitting=self.params.no_lane_splitting,
+                no_undetermined_fastqs=self.params.no_undetermined_fastqs,
                 create_fastq_for_index_read=\
                 self.params.create_fastq_for_index_read,
                 find_adapters_with_sliding_window=\
@@ -1086,6 +1089,11 @@ def _build_pipeline(self):
             create_fastq_for_index_read = \
                 subset['create_fastq_for_index_read']
 
+            #########################
+            # No undetermined Fastqs
+            #########################
+            no_undetermined_fastqs = subset['no_undetermined_fastqs']
+
             # Use sliding window for adapter trimming
             find_adapters_with_sliding_window = \
                 subset['find_adapters_with_sliding_window']
@@ -1200,6 +1208,7 @@ def _build_pipeline(self):
                         no_lane_splitting=self.params.no_lane_splitting,
                         create_fastq_for_index_read=\
                         create_fastq_for_index_read,
+                        no_undetermined_fastqs=no_undetermined_fastqs,
                         find_adapters_with_sliding_window=\
                         find_adapters_with_sliding_window,
                         create_empty_fastqs=self.params.create_empty_fastqs,
@@ -1284,6 +1293,7 @@ def _build_pipeline(self):
                                 create_fastq_for_index_read=\
                                 create_fastq_for_index_read,
                                 create_empty_fastqs=False,
+                                no_undetermined_fastqs=no_undetermined_fastqs,
                                 ignore_missing_fastqs=True,
                                 platform=identify_platform.output.platform,
                                 bclconvert_exe=\
@@ -1342,6 +1352,8 @@ def _build_pipeline(self):
                             create_fastq_for_index_read,
                             create_empty_fastqs=\
                             self.params.create_empty_fastqs,
+                            no_undetermined_fastqs=\
+                            self.params.no_undetermined_fastqs,
                             platform=identify_platform.output.platform,
                             bclconvert_exe=\
                             get_bclconvert.output.bclconvert_exe,
@@ -1684,9 +1696,9 @@ def run(self,analysis_dir,out_dir=None,barcode_analysis_dir=None,
             primary_data_dir=None,force_copy_of_primary_data=False,
             no_lane_splitting=None,create_fastq_for_index_read=None,
             find_adapters_with_sliding_window=None,
-            create_empty_fastqs=None,ignore_missing_bcls=None,
-            name=None,stats_file=None,stats_full=None,
-            per_lane_stats=None,per_lane_sample_stats=None,
+            create_empty_fastqs=None, no_undetermined_fastqs=None,
+            ignore_missing_bcls=None, name=None,stats_file=None,
+            stats_full=None, per_lane_stats=None,per_lane_sample_stats=None,
             nprocessors=None,cellranger_jobmode='local',
             cellranger_mempercore=None,cellranger_maxjobs=None,
             cellranger_jobinterval=None,cellranger_localcores=None,
@@ -1722,6 +1734,8 @@ def run(self,analysis_dir,out_dir=None,barcode_analysis_dir=None,
             sequences (--find-adapters-with-sliding-window)
           create_empty_fastqs (bool): if True then create empty
             "placeholder" Fastqs if not created by bcl2fastq
+          no_undetermined_fastqs (bool): if True then don't keep
+            the "undetermined" Fastqs if created
           ignore_missing_bcls (bool): if True then ignore missing
             or corrupted BCL files
           name (str): optional identifier for output
@@ -1905,6 +1919,7 @@ def run(self,analysis_dir,out_dir=None,barcode_analysis_dir=None,
             'find_adapters_with_sliding_window':
             find_adapters_with_sliding_window,
             'create_empty_fastqs': create_empty_fastqs,
+            'no_undetermined_fastqs': no_undetermined_fastqs,
             'ignore_missing_bcls': ignore_missing_bcls,
             'name': name,
             'stats_file': stats_file,
@@ -2391,7 +2406,7 @@ def init(self,run_dir,out_dir,sample_sheet,bases_mask=None,
              mask_short_adapter_reads=None,
              create_fastq_for_index_read=False,
              find_adapters_with_sliding_window=False,nprocessors=None,
-             create_empty_fastqs=False,
+             create_empty_fastqs=False, no_undetermined_fastqs=False,
              platform=None,bcl2fastq_exe=None,bcl2fastq_version=None,
              skip_bcl2fastq=False,conda_pkgs=None):
         """
@@ -2414,6 +2429,9 @@ def init(self,run_dir,out_dir,sample_sheet,bases_mask=None,
           create_fastq_for_index_read (boolean): if True then
             also create Fastq files for index reads (default,
             don't create index read Fastqs)
+          no_undetermined_fastqs (bool): if True then don't
+            keep any 'undetermined' Fastq files (default, do
+            keep the 'undetermined' Fastqs)
           find_adapters_with_sliding_window (bool): if True
             then use sliding window algorith for identifying
             adapter sequences (default is to use string
@@ -2600,6 +2618,17 @@ def finish(self):
                     # Terminate with an exception
                     raise Exception("Failed to verify outputs against "
                                     "samplesheet")
+            # Remove undetermined Fastqs
+            if self.args.no_undetermined_fastqs:
+                illumina_data = IlluminaData(os.path.dirname(self.tmp_out_dir),
+                                             os.path.basename(self.tmp_out_dir))
+                if illumina_data.undetermined:
+                    print("Removing undetermined fastqs")
+                    for undetermined_sample in illumina_data.undetermined.samples:
+                        for fq in undetermined_sample.fastq:
+                            fq = os.path.join(undetermined_sample.dirn, fq)
+                            if os.path.exists(fq):
+                                os.remove(fq)
             # Move to final location
             print("Moving output to final location: %s" % self.args.out_dir)
             os.rename(self.tmp_out_dir,self.args.out_dir)
@@ -2613,9 +2642,9 @@ def init(self,run_dir,out_dir,sample_sheet,lane=None,bases_mask=None,
              minimum_trimmed_read_length=None,
              mask_short_adapter_reads=None,
              create_fastq_for_index_read=False,nprocessors=None,
-             create_empty_fastqs=False,ignore_missing_fastqs=False,
-             platform=None,bclconvert_exe=None,bclconvert_version=None,
-             skip_bclconvert=False):
+             create_empty_fastqs=False, no_undetermined_fastqs=False,
+             ignore_missing_fastqs=False, platform=None, bclconvert_exe=None,
+             bclconvert_version=None, skip_bclconvert=False):
         """
         Initialise the RunBclConvert task
 
@@ -2636,6 +2665,8 @@ def init(self,run_dir,out_dir,sample_sheet,lane=None,bases_mask=None,
           create_fastq_for_index_read (boolean): if True then
             also create Fastq files for index reads (default,
             don't create index read Fastqs)
+          no_undetermined_fastqs (bool): if True then don't
+            create 'undetermined' Fastq files
           nprocessors (int): number of processors to use
             (taken from job runner by default)
           create_empty_fastqs (bool): if True then create empty
@@ -2869,6 +2900,17 @@ def finish(self):
                     # Terminate with an exception
                     raise Exception("Failed to verify outputs against "
                                     "samplesheet")
+            # Remove undetermined Fastqs
+            if self.args.no_undetermined_fastqs:
+                illumina_data = IlluminaData(os.path.dirname(self.tmp_out_dir),
+                                             os.path.basename(self.tmp_out_dir))
+                if illumina_data.undetermined:
+                    print("Removing undetermined fastqs")
+                    for undetermined_sample in illumina_data.undetermined.samples:
+                        for fq in undetermined_sample.fastq:
+                            fq = os.path.join(undetermined_sample.dirn, fq)
+                            if os.path.exists(fq):
+                                os.remove(fq)
             # Move to final location
             print("Moving output to final location: %s" % self.args.out_dir)
             os.rename(self.tmp_out_dir,self.args.out_dir)

auto_process_ngs/cli/auto_process.py

@@ -453,6 +453,13 @@ def add_make_fastqs_command(cmdparser):
                               dest='create_fastq_for_index_read',
                               default=False,
                               help="also create FASTQs for index reads")
+    # Don't include 'undetermined' Fastqs
+    bcl_to_fastq.add_argument('--no-undetermined-fastqs',
+                              action='store_true',
+                              dest='no_undetermined_fastqs',
+                              default=False,
+                              help="don't create or keept the 'undetermined' FASTQs "
+                              "with unassigned reads from demultiplexing")
     # Ignore missing or corrupted BCL files
     bcl_to_fastq.add_argument("--ignore-missing-bcls", action="store_true",
                               dest="ignore_missing_bcls", default=False,
@@ -1639,6 +1646,7 @@ def make_fastqs(args):
         adapter_sequence=args.adapter_sequence,
         adapter_sequence_read2=args.adapter_sequence_read2,
         create_fastq_for_index_read=args.create_fastq_for_index_read,
+        no_undetermined_fastqs=args.no_undetermined_fastqs,
         ignore_missing_bcls=args.ignore_missing_bcls,
         find_adapters_with_sliding_window=\
         args.find_adapters_with_sliding_window,

auto_process_ngs/commands/make_fastqs_cmd.py

@@ -53,6 +53,7 @@ def make_fastqs(ap,protocol='standard',platform=None,
                 analyse_barcodes=True,barcode_analysis_dir=None,
                 force_copy_of_primary_data=False,
                 create_empty_fastqs=False,
+                no_undetermined_fastqs=False,
                 ignore_missing_bcls=False,runner=None,
                 cellranger_jobmode=None,
                 cellranger_mempercore=None,
@@ -165,6 +166,8 @@ def make_fastqs(ap,protocol='standard',platform=None,
       create_empty_fastqs (bool): if True then create empty 'placeholder'
         fastq files for any missing fastqs after bcl2fastq
         (must have completed with zero exit status)
+      no_undetermined_fastqs (bool): if True then don't keep the
+        "undetermined" fastq files
       runner (JobRunner): (optional) specify a non-default job runner
         to use for fastq generation
       cellranger_jobmode (str): (optional) job mode to run cellranger in
@@ -440,6 +443,7 @@ def make_fastqs(ap,protocol='standard',platform=None,
                              find_adapters_with_sliding_window,
                              create_empty_fastqs=create_empty_fastqs,
                              ignore_missing_bcls=ignore_missing_bcls,
+                             no_undetermined_fastqs=no_undetermined_fastqs,
                              stats_file=stats_file,
                              per_lane_stats=per_lane_stats_file,
                              nprocessors=nprocessors,

auto_process_ngs/test/bcl2fastq/pipeline/test_standard_bcl2fastq.py

@@ -1032,6 +1032,86 @@ def test_makefastqs_standard_protocol_multiple_lanes_reduced_subset(self):
                 os.path.join(analysis_dir,filen)),
                             "Missing file: %s" % filen)
 
+    #@unittest.skip("Skipped")
+    def test_makefastqs_standard_protocol_no_undetermined_fastqs(self):
+        """
+        MakeFastqs: standard protocol/bcl2fastq: no 'undetermined' Fastqs
+        """
+        # Create mock source data
+        illumina_run = MockIlluminaRun(
+            "171020_M00879_00002_AHGXXXX",
+            "miseq",
+            top_dir=self.wd)
+        illumina_run.create()
+        run_dir = illumina_run.dirn
+        # Sample sheet
+        sample_sheet = os.path.join(self.wd,"SampleSheet.csv")
+        with open(sample_sheet,'wt') as fp:
+            fp.write(SampleSheets.miseq)
+        # Create mock bcl2fastq
+        # Check that --create-fastq-for-index-read is set
+        MockBcl2fastq2Exe.create(os.path.join(self.bin, "bcl2fastq"))
+        os.environ['PATH'] = "%s:%s" % (self.bin,
+                                        os.environ['PATH'])
+        # Make an (empty) analysis directory
+        analysis_dir = os.path.join(self.wd,"analysis")
+        os.mkdir(analysis_dir)
+        # Do the test
+        p = MakeFastqs(run_dir,sample_sheet)
+        status = p.run(analysis_dir,
+                       no_undetermined_fastqs=True,
+                       poll_interval=POLL_INTERVAL)
+        self.assertEqual(status,0)
+        # Check outputs
+        self.assertEqual(p.output.platform,"miseq")
+        self.assertEqual(p.output.flow_cell_mode,None)
+        self.assertEqual(p.output.primary_data_dir,
+                         os.path.join(analysis_dir,
+                                      "primary_data"))
+        self.assertEqual(p.output.bcl2fastq_info,
+                         (os.path.join(self.bin,"bcl2fastq"),
+                          "bcl2fastq",
+                          "2.20.0.422"))
+        self.assertEqual(p.output.cellranger_info,None)
+        self.assertTrue(p.output.acquired_primary_data)
+        self.assertEqual(p.output.stats_file,
+                         os.path.join(analysis_dir,"statistics.info"))
+        self.assertEqual(p.output.stats_full,
+                         os.path.join(analysis_dir,"statistics_full.info"))
+        self.assertEqual(p.output.per_lane_stats,
+                         os.path.join(analysis_dir,
+                                      "per_lane_statistics.info"))
+        self.assertEqual(p.output.per_lane_sample_stats,
+                         os.path.join(analysis_dir,
+                                      "per_lane_sample_stats.info"))
+        self.assertEqual(p.output.seq_len_stats,
+                         os.path.join(analysis_dir,
+                                      "seq_len_statistics.info"))
+        self.assertEqual(p.output.missing_fastqs,[])
+        for subdir in (os.path.join("primary_data",
+                                    "171020_M00879_00002_AHGXXXX"),
+                       "bcl2fastq",
+                       "barcode_analysis",):
+            self.assertTrue(os.path.isdir(
+                os.path.join(analysis_dir,subdir)),
+                            "Missing subdir: %s" % subdir)
+        self.assertTrue(os.path.islink(
+            os.path.join(analysis_dir,
+                         "primary_data",
+                         "171020_M00879_00002_AHGXXXX")))
+        for filen in ("statistics.info",
+                      "statistics_full.info",
+                      "per_lane_statistics.info",
+                      "per_lane_sample_stats.info",
+                      "seq_len_statistics.info",
+                      "processing_qc.html"):
+            self.assertTrue(os.path.isfile(
+                os.path.join(analysis_dir,filen)),
+                            "Missing file: %s" % filen)
+        for fq in ["Undetermined_S0_L001_R2_001.fastq.gz"]:
+            self.assertFalse(os.path.exists(os.path.join(analysis_dir, "bcl2fastq", fq)),
+                             f"'{fq}' is present (should be absent)")
+
     #@unittest.skip("Skipped")
     def test_makefastqs_standard_protocol_rerun_completed_pipeline(self):
         """