Merge pull request #1179 from fls-bioinformatics-core/transfer_data-shorten-shared-filenames

'transfer_data.py': add option to shorten shared filenames

Author: pjbriggs

auto_process_ngs/cli/transfer_data.py

@@ -291,6 +291,10 @@ def main(argv=None):
                    help="copy the 'download_fastqs.py' utility to the "
                     "final location")
     sp = p.add_argument_group("Advanced options")
+    sp.add_argument('--short_names',action='store_true',
+                    default=None,
+                    help="use shortened base names where possible for "
+                    "shared files")
     sp.add_argument('--link',action='store_true',
                     help="hard link files instead of copying")
     sp.add_argument('--runner',action='store',
@@ -337,6 +341,7 @@ def main(argv=None):
         subdir = dest.subdir
         zip_fastqs = dest.zip_fastqs
         max_zip_size = dest.max_zip_size
+        short_names = dest.short_names
         include_downloader = dest.include_downloader
         include_qc_report = dest.include_qc_report
         hard_links = dest.hard_links
@@ -347,6 +352,7 @@ def main(argv=None):
         subdir = None
         zip_fastqs = False
         max_zip_size = None
+        short_names = False
         include_downloader = False
         include_qc_report = False
         hard_links = False
@@ -361,6 +367,8 @@ def main(argv=None):
         zip_fastqs = True
     if args.max_zip_size:
         max_zip_size = args.max_zip_size
+    if args.short_names:
+        short_names = args.short_names
     if args.include_downloader:
         include_downloader = True
     if args.include_qc_report:
@@ -392,6 +400,7 @@ def main(argv=None):
     print(f"Hard link Fastqs     : {hard_links}")
     print(f"Zip Fastqs           : {zip_fastqs}")
     print(f"Max ZIP size         : {max_zip_size}")
+    print(f"Shorten file names   : {short_names}")
     print(f"Dry run              : {dry_run}")
 
     # Check at least one artefact is being transferred
@@ -741,6 +750,7 @@ def main(argv=None):
                     analysis_dir.metadata.run_number,
                     analysis_dir.metadata.instrument_datestamp,
                     analysis_dir.run_id))
+    summary.append(f"Project '{project.name}'")
     summary.append("%s%s dataset" %
                    ("%s " % project.info.single_cell_platform
                     if project.info.single_cell_platform else '',
@@ -857,13 +867,16 @@ def main(argv=None):
             run_number = str(analysis_dir.metadata.run_number)
             if analysis_dir.metadata.analysis_number is not None:
                 run_number += "_" + str(analysis_dir.metadata.analysis_number)
-            final_zip_basename = \
-                "{platform}_{datestamp}.{run_number}-{project}-fastqs".\
-                format(
-                    platform=analysis_dir.metadata.platform.upper(),
-                    datestamp=analysis_dir.metadata.instrument_datestamp,
-                    run_number=run_number,
-                    project=project.name)
+            if short_names:
+                final_zip_basename = f"{project.name}-fastqs"
+            else:
+                final_zip_basename = \
+                    "{platform}_{datestamp}.{run_number}-{project}-fastqs".\
+                    format(
+                        platform=analysis_dir.metadata.platform.upper(),
+                        datestamp=analysis_dir.metadata.instrument_datestamp,
+                        run_number=run_number,
+                        project=project.name)
             job_ix = 0
             for f in listdir(working_dir):
                 if f == "%s.chksums" % project_name:
@@ -907,11 +920,14 @@ def main(argv=None):
     if qc_zips:
         for qc_zip in qc_zips:
             print("Copying '%s'" % os.path.basename(qc_zip))
+            qc_zip_basename = os.path.basename(qc_zip)
+            if short_names:
+                # Hack to remove run name from QC report ZIP name
+                qc_zip_basename = f"{'.'.join(qc_zip_basename.split('.')[:-2])}.zip"
             td.run_job(
-                f"copy_qc_zip.{job_id}.{os.path.basename(qc_zip)}",
+                f"copy_qc_zip.{job_id}.{qc_zip_basename}",
                 copy_command(qc_zip,
-                             os.path.join(target_dir,
-                                          os.path.basename(qc_zip)),
+                             os.path.join(target_dir, qc_zip_basename),
                              link=hard_links))
 
     # Tar and copy 10xGenomics outputs
@@ -920,12 +936,13 @@ def main(argv=None):
             print("Tar gzipping and copying '%s'" %
                   os.path.basename(cellranger_dir))
             # Tar & gzip data
-            targz = os.path.join(working_dir,
-                                 "%s.%s.%s.tgz" % (
-                                     os.path.basename(
-                                         cellranger_dir),
-                                     project_name,
-                                     project.info.run))
+            if short_names:
+                targz = f"{os.path.basename(cellranger_dir)}.{project_name}.tgz"
+            else:
+                targz = "%s.%s.%s.tgz" % (os.path.basename(cellranger_dir),
+                                          project_name,
+                                          project.info.run)
+            targz = os.path.join(working_dir, targz)
             targz_job = td.run_job(
                 f"targz_10x_output.{job_id}.{os.path.basename(cellranger_dir)}",
                 Command("tar",
@@ -980,11 +997,13 @@ def main(argv=None):
     if visium_images_dir:
         print(f"Tar gzipping and copying '{visium_images_dir}'")
         # Tar & gzip data
-        targz = os.path.join(working_dir,
-                             "%s.%s.%s.tgz" % (
-                                 os.path.basename(visium_images_dir),
-                                 project_name,
-                                 project.info.run))
+        if short_names:
+            targz = f"{os.path.basename(visium_images_dir)}.{project_name}.tgz"
+        else:
+            targz = "%s.%s.%s.tgz" % (os.path.basename(visium_images_dir),
+                                      project_name,
+                                      project.info.run)
+        targz = os.path.join(working_dir, targz)
         targz_job = td.run_job(
             f"targz_visium_images.{job_id}.{os.path.basename(visium_images_dir)}",
             Command("tar",

auto_process_ngs/settings.py

@@ -1060,6 +1060,7 @@ def __init__(self, settings_file=None, resolve_undefined=True):
                                    "subdir": str,
                                    "zip_fastqs": bool,
                                    "max_zip_size": str,
+                                   "short_names": bool,
                                    "readme_template": str,
                                    "url": str,
                                    "include_downloader": bool,
@@ -1083,6 +1084,7 @@ def __init__(self, settings_file=None, resolve_undefined=True):
                 "destination:*.include_qc_report": False,
                 "destination:*.hard_links": False,
                 "destination:*.zip_fastqs": False,
+                "destination:*.short_names": False,
                 "destination:*.include_downloader": False,
                 "qc.fastq_subset_size": 100000,
                 "qc.split_undetermined_fastqs": True,

auto_process_ngs/test/cli/test_transfer_data.py

@@ -664,3 +664,78 @@ def test_transfer_data_dry_run(self):
              "--dry-run" ]), 0)
         # Check nothing was transferred
         self.assertEqual(len(os.listdir(target_dir)), 0)
+
+    def test_transfer_data_fastqs_with_qc_report_short_names(self):
+        """
+        transfer_data: copy Fastqs and QC report with shortened names
+        """
+        # Make a mock auto-process directory
+        mockdir = MockAnalysisDirFactory.bcl2fastq2(
+            '170901_M00879_0087_000000000-AGEW9',
+            'miseq',
+            metadata={ "instrument_datestamp": "170901",
+                       "run_number": "89" },
+            project_metadata={ "AB": { "Library type": "RNA-seq",
+                                       "Organism": "Human" } },
+            top_dir=self.dirn)
+        mockdir.create()
+        # Add QC outputs
+        project = AnalysisProject(os.path.join(mockdir.dirn, "AB"))
+        UpdateAnalysisProject(project).add_qc_outputs()
+        # Make a target directory
+        target_dir = os.path.join(self.dirn, "shared")
+        os.makedirs(target_dir)
+        # Do data transfer (--include_qc_report)
+        self.assertEqual(transfer_data(
+            [target_dir,
+             os.path.join(mockdir.dirn, "AB"),
+             "--include_qc_report",
+             "--short_names"]), 0)
+        # Check transferred artefacts
+        print(os.listdir(target_dir))
+        expected_files = ("AB1_S1_R1_001.fastq.gz",
+                          "AB1_S1_R2_001.fastq.gz",
+                          "AB2_S2_R2_001.fastq.gz",
+                          "AB2_S2_R1_001.fastq.gz",
+                          "AB.chksums",
+                          "qc_report.AB.zip")
+        for f in expected_files:
+            self.assertTrue(os.path.exists(os.path.join(target_dir, f)),
+                            f"'{f}': missing, should be present")
+        for f in os.listdir(target_dir):
+            self.assertTrue(f in expected_files,
+                            f"'{f}': present, but not expected")
+
+    def test_transfer_data_zip_fastqs_short_names(self):
+        """
+        transfer_data: put Fastqs into ZIP archive using shortened names
+        """
+        # Make a mock auto-process directory
+        mockdir = MockAnalysisDirFactory.bcl2fastq2(
+            '170901_M00879_0087_000000000-AGEW9',
+            'miseq',
+            metadata={ "instrument_datestamp": "170901",
+                       "run_number": "89" },
+            project_metadata={ "AB": { "Library type": "RNA-seq",
+                                       "Organism": "Human" } },
+            top_dir=self.dirn)
+        mockdir.create()
+        # Make a target directory
+        target_dir = os.path.join(self.dirn, "shared")
+        os.makedirs(target_dir)
+        # Do data transfer (--zip_fastqs)
+        self.assertEqual(transfer_data(
+            [target_dir,
+             os.path.join(mockdir.dirn, "AB"),
+             "--zip_fastqs",
+             "--short_names"]), 0)
+        # Check transferred artefacts
+        print(os.listdir(target_dir))
+        expected_files = ("AB-fastqs.zip",
+                          "AB-fastqs.checksums")
+        for f in expected_files:
+            self.assertTrue(os.path.exists(os.path.join(target_dir, f)),
+                            f"'{f}': missing, should be present")
+        for f in os.listdir(target_dir):
+            self.assertTrue(f in expected_files,
+                            f"'{f}': present, but not expected")

config/auto_process.ini.sample

@@ -182,6 +182,7 @@
 #   * 'run_id': creates new subdir 'PLATFORM_DATESTAMP.RUN_ID-PROJECT'
 # - zip_fastqs: whether to bundle Fastqs into ZIP archives
 # - max_zip_size: maximum size for each ZIP archive
+# - short_names: use shorter naming scheme for shared files
 # - readme_template: template file to generate README file from
 #   (either full path or the name of a file in the 'templates'
 #   directory of the installation
@@ -199,6 +200,7 @@
 ;subdir = random_bin
 ;zip_fastqs = true
 ;max_zip_size = 5G
+;short_names = true
 ;readme_template = README.webserver.txt
 ;url = http://awesome.com/data
 ;include_downloader = true

docs/source/configuration.rst

@@ -722,8 +722,10 @@ Parameter              Function
 ``zip_fastqs``         Whether to bundle Fastqs into ZIP archives
 ``max_zip_size``       Maximum size for each ZIP archive (if Fastqs
                        are bundled)
+``short_names``        Use shortened base name where possible for the
+                       shared files
 ``readme_template``    Template file to generate ``README`` from
-``url``                Base URL to access copied data at
+``url``                Base URL to access copied data from
 ``include_downloader`` Whether to include ``download_fastqs.py``
 ``include_qc_report``  Whether to include zipped QC reports
 ``hard_links``         Whether to hard link to Fastqs rather making