diff --git a/auto_process_ngs/cli/transfer_data.py b/auto_process_ngs/cli/transfer_data.py index 6ac0035b6..e2bb636b5 100644 --- a/auto_process_ngs/cli/transfer_data.py +++ b/auto_process_ngs/cli/transfer_data.py @@ -291,6 +291,10 @@ def main(argv=None): help="copy the 'download_fastqs.py' utility to the " "final location") sp = p.add_argument_group("Advanced options") + sp.add_argument('--short_names',action='store_true', + default=None, + help="use shortened base names where possible for " + "shared files") sp.add_argument('--link',action='store_true', help="hard link files instead of copying") sp.add_argument('--runner',action='store', @@ -337,6 +341,7 @@ def main(argv=None): subdir = dest.subdir zip_fastqs = dest.zip_fastqs max_zip_size = dest.max_zip_size + short_names = dest.short_names include_downloader = dest.include_downloader include_qc_report = dest.include_qc_report hard_links = dest.hard_links @@ -347,6 +352,7 @@ def main(argv=None): subdir = None zip_fastqs = False max_zip_size = None + short_names = False include_downloader = False include_qc_report = False hard_links = False @@ -361,6 +367,8 @@ def main(argv=None): zip_fastqs = True if args.max_zip_size: max_zip_size = args.max_zip_size + if args.short_names: + short_names = args.short_names if args.include_downloader: include_downloader = True if args.include_qc_report: @@ -392,6 +400,7 @@ def main(argv=None): print(f"Hard link Fastqs : {hard_links}") print(f"Zip Fastqs : {zip_fastqs}") print(f"Max ZIP size : {max_zip_size}") + print(f"Shorten file names : {short_names}") print(f"Dry run : {dry_run}") # Check at least one artefact is being transferred @@ -741,6 +750,7 @@ def main(argv=None): analysis_dir.metadata.run_number, analysis_dir.metadata.instrument_datestamp, analysis_dir.run_id)) + summary.append(f"Project '{project.name}'") summary.append("%s%s dataset" % ("%s " % project.info.single_cell_platform if project.info.single_cell_platform else '', @@ -857,13 +867,16 @@ def main(argv=None): run_number = str(analysis_dir.metadata.run_number) if analysis_dir.metadata.analysis_number is not None: run_number += "_" + str(analysis_dir.metadata.analysis_number) - final_zip_basename = \ - "{platform}_{datestamp}.{run_number}-{project}-fastqs".\ - format( - platform=analysis_dir.metadata.platform.upper(), - datestamp=analysis_dir.metadata.instrument_datestamp, - run_number=run_number, - project=project.name) + if short_names: + final_zip_basename = f"{project.name}-fastqs" + else: + final_zip_basename = \ + "{platform}_{datestamp}.{run_number}-{project}-fastqs".\ + format( + platform=analysis_dir.metadata.platform.upper(), + datestamp=analysis_dir.metadata.instrument_datestamp, + run_number=run_number, + project=project.name) job_ix = 0 for f in listdir(working_dir): if f == "%s.chksums" % project_name: @@ -907,11 +920,14 @@ def main(argv=None): if qc_zips: for qc_zip in qc_zips: print("Copying '%s'" % os.path.basename(qc_zip)) + qc_zip_basename = os.path.basename(qc_zip) + if short_names: + # Hack to remove run name from QC report ZIP name + qc_zip_basename = f"{'.'.join(qc_zip_basename.split('.')[:-2])}.zip" td.run_job( - f"copy_qc_zip.{job_id}.{os.path.basename(qc_zip)}", + f"copy_qc_zip.{job_id}.{qc_zip_basename}", copy_command(qc_zip, - os.path.join(target_dir, - os.path.basename(qc_zip)), + os.path.join(target_dir, qc_zip_basename), link=hard_links)) # Tar and copy 10xGenomics outputs @@ -920,12 +936,13 @@ def main(argv=None): print("Tar gzipping and copying '%s'" % os.path.basename(cellranger_dir)) # Tar & gzip data - targz = os.path.join(working_dir, - "%s.%s.%s.tgz" % ( - os.path.basename( - cellranger_dir), - project_name, - project.info.run)) + if short_names: + targz = f"{os.path.basename(cellranger_dir)}.{project_name}.tgz" + else: + targz = "%s.%s.%s.tgz" % (os.path.basename(cellranger_dir), + project_name, + project.info.run) + targz = os.path.join(working_dir, targz) targz_job = td.run_job( f"targz_10x_output.{job_id}.{os.path.basename(cellranger_dir)}", Command("tar", @@ -980,11 +997,13 @@ def main(argv=None): if visium_images_dir: print(f"Tar gzipping and copying '{visium_images_dir}'") # Tar & gzip data - targz = os.path.join(working_dir, - "%s.%s.%s.tgz" % ( - os.path.basename(visium_images_dir), - project_name, - project.info.run)) + if short_names: + targz = f"{os.path.basename(visium_images_dir)}.{project_name}.tgz" + else: + targz = "%s.%s.%s.tgz" % (os.path.basename(visium_images_dir), + project_name, + project.info.run) + targz = os.path.join(working_dir, targz) targz_job = td.run_job( f"targz_visium_images.{job_id}.{os.path.basename(visium_images_dir)}", Command("tar", diff --git a/auto_process_ngs/settings.py b/auto_process_ngs/settings.py index f2a705ded..cecf763a2 100644 --- a/auto_process_ngs/settings.py +++ b/auto_process_ngs/settings.py @@ -1060,6 +1060,7 @@ def __init__(self, settings_file=None, resolve_undefined=True): "subdir": str, "zip_fastqs": bool, "max_zip_size": str, + "short_names": bool, "readme_template": str, "url": str, "include_downloader": bool, @@ -1083,6 +1084,7 @@ def __init__(self, settings_file=None, resolve_undefined=True): "destination:*.include_qc_report": False, "destination:*.hard_links": False, "destination:*.zip_fastqs": False, + "destination:*.short_names": False, "destination:*.include_downloader": False, "qc.fastq_subset_size": 100000, "qc.split_undetermined_fastqs": True, diff --git a/auto_process_ngs/test/cli/test_transfer_data.py b/auto_process_ngs/test/cli/test_transfer_data.py index eba10de10..188c2932c 100644 --- a/auto_process_ngs/test/cli/test_transfer_data.py +++ b/auto_process_ngs/test/cli/test_transfer_data.py @@ -664,3 +664,78 @@ def test_transfer_data_dry_run(self): "--dry-run" ]), 0) # Check nothing was transferred self.assertEqual(len(os.listdir(target_dir)), 0) + + def test_transfer_data_fastqs_with_qc_report_short_names(self): + """ + transfer_data: copy Fastqs and QC report with shortened names + """ + # Make a mock auto-process directory + mockdir = MockAnalysisDirFactory.bcl2fastq2( + '170901_M00879_0087_000000000-AGEW9', + 'miseq', + metadata={ "instrument_datestamp": "170901", + "run_number": "89" }, + project_metadata={ "AB": { "Library type": "RNA-seq", + "Organism": "Human" } }, + top_dir=self.dirn) + mockdir.create() + # Add QC outputs + project = AnalysisProject(os.path.join(mockdir.dirn, "AB")) + UpdateAnalysisProject(project).add_qc_outputs() + # Make a target directory + target_dir = os.path.join(self.dirn, "shared") + os.makedirs(target_dir) + # Do data transfer (--include_qc_report) + self.assertEqual(transfer_data( + [target_dir, + os.path.join(mockdir.dirn, "AB"), + "--include_qc_report", + "--short_names"]), 0) + # Check transferred artefacts + print(os.listdir(target_dir)) + expected_files = ("AB1_S1_R1_001.fastq.gz", + "AB1_S1_R2_001.fastq.gz", + "AB2_S2_R2_001.fastq.gz", + "AB2_S2_R1_001.fastq.gz", + "AB.chksums", + "qc_report.AB.zip") + for f in expected_files: + self.assertTrue(os.path.exists(os.path.join(target_dir, f)), + f"'{f}': missing, should be present") + for f in os.listdir(target_dir): + self.assertTrue(f in expected_files, + f"'{f}': present, but not expected") + + def test_transfer_data_zip_fastqs_short_names(self): + """ + transfer_data: put Fastqs into ZIP archive using shortened names + """ + # Make a mock auto-process directory + mockdir = MockAnalysisDirFactory.bcl2fastq2( + '170901_M00879_0087_000000000-AGEW9', + 'miseq', + metadata={ "instrument_datestamp": "170901", + "run_number": "89" }, + project_metadata={ "AB": { "Library type": "RNA-seq", + "Organism": "Human" } }, + top_dir=self.dirn) + mockdir.create() + # Make a target directory + target_dir = os.path.join(self.dirn, "shared") + os.makedirs(target_dir) + # Do data transfer (--zip_fastqs) + self.assertEqual(transfer_data( + [target_dir, + os.path.join(mockdir.dirn, "AB"), + "--zip_fastqs", + "--short_names"]), 0) + # Check transferred artefacts + print(os.listdir(target_dir)) + expected_files = ("AB-fastqs.zip", + "AB-fastqs.checksums") + for f in expected_files: + self.assertTrue(os.path.exists(os.path.join(target_dir, f)), + f"'{f}': missing, should be present") + for f in os.listdir(target_dir): + self.assertTrue(f in expected_files, + f"'{f}': present, but not expected") diff --git a/config/auto_process.ini.sample b/config/auto_process.ini.sample index 8ae9f392f..31c5f36b3 100644 --- a/config/auto_process.ini.sample +++ b/config/auto_process.ini.sample @@ -182,6 +182,7 @@ # * 'run_id': creates new subdir 'PLATFORM_DATESTAMP.RUN_ID-PROJECT' # - zip_fastqs: whether to bundle Fastqs into ZIP archives # - max_zip_size: maximum size for each ZIP archive +# - short_names: use shorter naming scheme for shared files # - readme_template: template file to generate README file from # (either full path or the name of a file in the 'templates' # directory of the installation @@ -199,6 +200,7 @@ ;subdir = random_bin ;zip_fastqs = true ;max_zip_size = 5G +;short_names = true ;readme_template = README.webserver.txt ;url = http://awesome.com/data ;include_downloader = true diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index c69b88d17..0234b66ec 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -722,8 +722,10 @@ Parameter Function ``zip_fastqs`` Whether to bundle Fastqs into ZIP archives ``max_zip_size`` Maximum size for each ZIP archive (if Fastqs are bundled) +``short_names`` Use shortened base name where possible for the + shared files ``readme_template`` Template file to generate ``README`` from -``url`` Base URL to access copied data at +``url`` Base URL to access copied data from ``include_downloader`` Whether to include ``download_fastqs.py`` ``include_qc_report`` Whether to include zipped QC reports ``hard_links`` Whether to hard link to Fastqs rather making