Merge pull request #1154 from fls-bioinformatics-core/update-cmd-handle-inconsistent-projects.info

'update': extend to handle inconsistencies between 'projects.info' and contents of analysis directory

Author: pjbriggs

auto_process_ngs/cli/auto_process.py

@@ -1062,6 +1062,18 @@ def add_update_command(cmdparser):
                               "ANALYSIS_DIR and its projects and QC outputs "
                               "when directory has been moved or copied, "
                               "or project metadata has been updated.")
+    p.add_argument('--paths', action='store_true',
+                   help="update paths stored in the metadata and parameter "
+                        "files for ANALYSIS_DIR")
+    p.add_argument('--project-metadata', action='store_true',
+                   help="propagate modified metadata in 'projects.info' to "
+                   "project directories in ANALYSIS_DIR")
+    p.add_argument('--project-dirs', action='store_true',
+                   help="synchronise project entries in 'projects.info' with "
+                   "project directories within ANALYSIS_DIR")
+    p.add_argument('--qc-reports', action='store_true',
+                   help="regenerate QC reports for projects where metadata "
+                   "has been updated")
     add_debug_option(p)
     p.add_argument('analysis_dir',metavar="ANALYSIS_DIR",nargs='?',
                    help="existing auto_process analysis directory to "
@@ -1835,7 +1847,10 @@ def update(args):
     if not analysis_dir:
         analysis_dir = os.getcwd()
     d = AutoProcess(analysis_dir)
-    d.update()
+    d.update(update_paths=args.paths,
+             update_project_metadata=args.project_metadata,
+             update_sync_projects=args.project_dirs,
+             update_qc_reports=args.qc_reports)
 
 def readme(args):
     """

auto_process_ngs/commands/update_cmd.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 #
 #     update_cmd.py: implement 'update' command
-#     Copyright (C) University of Manchester 2023 Peter Briggs
+#     Copyright (C) University of Manchester 2023-2026 Peter Briggs
 #
 #########################################################################
 
@@ -24,17 +24,32 @@
 # Command functions
 #######################################################################
 
-def update(ap):
+def update(ap, update_paths=True, update_project_metadata=True,
+           update_sync_projects=True, update_qc_reports=True):
     """
     Update metadata and artefacts in analysis directory
 
     Arguments:
       ap (AutoProcessor): autoprocessor pointing to the
         analysis directory to publish QC for
+      update_paths (bool): whether to update analysis
+        directory paths in metadata and parameter files
+        (default: True)
+      update_project_metadata (bool): whether to update
+        metadata stored in 'projects.info' and in the
+        project directories (default: True)
+      update_sync_projects (bool): whether to update
+        projects listed in 'projects.info' against
+        project directories on the filesystem (default:
+        True)
+      update_qc_reports (bool): whether to update QC
+        reports in projects where existing report is
+        older than the project metadata file (default:
+        True)
     """
-    update_paths = True
-    update_projects = True
-    update_qc_reports = True
+    if not (update_paths or update_project_metadata or
+            update_sync_projects or update_qc_reports):
+        logger.warning("No updates requested")
 
     if update_paths:
         # Update paths in the top-level parameter file
@@ -67,7 +82,57 @@ def update(ap):
             # Save the updated parameter data
             ap.save_parameters(force=True)
 
-    if update_projects:
+    if update_sync_projects:
+        # Load information from 'projects.info'
+        project_metadata = ap.load_project_metadata(
+            ap.params.project_metadata)
+        # Comment out projects which don't exist on filesystem
+        save_required = False
+        for line in project_metadata:
+            # Iterate through the named projects
+            name = line['Project']
+            if name.startswith('#'):
+                # Commented out, ignore
+                continue
+            # Look for a matching project directory
+            project_dir = os.path.join(ap.analysis_dir, name)
+            if not os.path.exists(project_dir):
+                print(f"Commenting out missing project '{name}'")
+                line['Project'] = f"#{name}"
+                save_required = True
+        if save_required:
+            project_metadata.save()
+        # Add any project directories without entries
+        save_required = False
+        projects = [line['Project'] for line in project_metadata]
+        for project in ap.get_analysis_projects_from_dirs():
+            if project.name.endswith(".bak") \
+                    or project.name.endswith(".orig") \
+                    or project.name.endswith(".tmp"):
+                # Skip directories with extensions indicating they
+                # should be ignored
+                print(f"Not adding entry for unlisted project '{project.name}'")
+                continue
+            elif project.name == "undetermined":
+                # Skip undetermined
+                continue
+            elif project.name not in projects and f"#{project.name}" not in projects:
+                # Add new entry
+                print(f"Adding entry for unlisted project '{project.name}'")
+                project_metadata.add_project(project.name,
+                                             [s.name for s in project.samples],
+                                             user=project.info.user,
+                                             PI=project.info.PI,
+                                             organism=project.info.organism,
+                                             library_type=project.info.library_type,
+                                             sc_platform=project.info.single_cell_platform,
+                                             comments=project.info.comments)
+                save_required = True
+        # Save the updated project metadata if required
+        if save_required:
+            project_metadata.save()
+
+    if update_project_metadata:
         # Update project metadata
         project_metadata = ap.load_project_metadata(
             ap.params.project_metadata)

auto_process_ngs/test/commands/test_update_cmd.py

@@ -78,7 +78,11 @@ def test_update_relocated_analysis_dir(self):
         self.assertEqual(ap.params.primary_data_dir,
                          os.path.join(original_path,"primary_data"))
         # Update the metadata
-        update(ap)
+        update(ap,
+               update_paths=True,
+               update_sync_projects=False,
+               update_project_metadata=False,
+               update_qc_reports=False)
         # Reload and check metadata items post-update
         ap = AutoProcess(new_path)
         self.assertEqual(ap.analysis_dir,new_path)
@@ -138,7 +142,11 @@ def test_update_relocated_analysis_dir_with_qc(self):
                                           proj.name,
                                           "fastqs"))
         # Update the metadata
-        update(ap)
+        update(ap,
+               update_paths=True,
+               update_sync_projects=False,
+               update_project_metadata=False,
+               update_qc_reports=False)
         # Reload and check metadata items post-update
         ap = AutoProcess(new_path)
         self.assertEqual(ap.analysis_dir,new_path)
@@ -215,7 +223,11 @@ def test_update_project_metadata_changed(self):
 AB\tAB1,AB2\tAlan Bailey\tRNA-seq\t.\tMouse\tArchie Ballard\t1% PhiX spiked in
 CDE\tCDE3,CDE4\tCharles Edwards\tChIP-seq\t.\tMouse\tChristian Eggars\t1% PhiX spiked in
 """)
-        update(ap)
+        update(ap,
+               update_paths=False,
+               update_sync_projects=False,
+               update_project_metadata=True,
+               update_qc_reports=False)
         # Reload and confirm the updates in projects.info
         ap = AutoProcess(mockdir.dirn)
         ap_project_metadata = ap.load_project_metadata()
@@ -291,7 +303,11 @@ def test_update_sample_list_changed(self):
             "CDE": ["CDE3"]
         }
         # Update sample lists in metadata
-        update(ap)
+        update(ap,
+               update_paths=False,
+               update_sync_projects=False,
+               update_project_metadata=True,
+               update_qc_reports=False)
         # Reload and check updated sample lists in projects.info
         ap = AutoProcess(mockdir.dirn)
         ap_project_metadata = ap.load_project_metadata()
@@ -337,7 +353,11 @@ def test_update_paired_end_changed(self):
             os.remove(os.path.join(mockdir.dirn,fq))
         # Set up AutoProcess instance and update metadata
         ap = AutoProcess(mockdir.dirn)
-        update(ap)
+        update(ap,
+               update_paths=False,
+               update_sync_projects=False,
+               update_project_metadata=True,
+               update_qc_reports=False)
         # Reload and check update paired-end metadata in projects
         ap = AutoProcess(mockdir.dirn)
         expected_paired_end = {
@@ -350,6 +370,176 @@ def test_update_paired_end_changed(self):
             self.assertEqual(project_metadata.paired_end,
                              expected_paired_end[pname])
 
+    def test_update_project_metadata_project_no_longer_exists(self):
+        """
+        update: handle project which no longer exists in analysis dir
+        """
+        # Metadata for projects
+        project_metadata = {
+            "AB": {
+                "User": "Alan Bailey",
+                "PI": "Archie Ballard",
+                "Library": "RNA-seq",
+                "Organism": "Human"
+            }
+        }
+        # Make an auto-process directory with projects
+        mockdir = MockAnalysisDirFactory.bcl2fastq2(
+            '231021_A00879_0087_000000000-AGEW9',
+            'novaseq',
+            project_metadata=project_metadata,
+            metadata={ "run_number": 87,
+                       "source": "local" },
+            top_dir=self.dirn)
+        mockdir.create()
+        # Remove CDE project
+        shutil.rmtree(os.path.join(mockdir.dirn, "CDE"))
+        # Remove initial entry for CDE in projects.info
+        projects_info_contents = []
+        with open(os.path.join(mockdir.dirn,"projects.info"),'rt') as fp:
+            for line in fp:
+                if not line.startswith("CDE"):
+                    projects_info_contents.append(line)
+        with open(os.path.join(mockdir.dirn,"projects.info"),'wt') as fp:
+            fp.write("".join(projects_info_contents))
+        # Set up AutoProcess instance
+        ap = AutoProcess(mockdir.dirn)
+        # Check metadata items in projects.info pre-update
+        ap_project_metadata = ap.load_project_metadata()
+        for pname in project_metadata:
+            for item in project_metadata[pname]:
+                self.assertEqual(ap_project_metadata.lookup(pname)[item],
+                                 project_metadata[pname][item])
+        # Check metadata items in projects pre-update
+        for pname in project_metadata:
+            p = ap.get_analysis_projects(pname)[0]
+            for item in project_metadata[pname]:
+                project_item = self.metadata_map[item]
+                expected_value = project_metadata[pname][item]
+                self.assertEqual(p.info[project_item],
+                                 expected_value)
+        # Append info for missing project 'CDE' to projects.info
+        with open(os.path.join(mockdir.dirn,"projects.info"),'at') as fp:
+            fp.write("""CDE\tCDE3,CDE4\tCharles Edwards\tChIP-seq\t.\tMouse\tChristian Eggars\t1% PhiX spiked in
+""")
+        # Do the update
+        update(ap,
+               update_paths=False,
+               update_sync_projects=True,
+               update_project_metadata=False,
+               update_qc_reports=False)
+        # Reload and confirm the updates in projects.info
+        ap = AutoProcess(mockdir.dirn)
+        ap_project_metadata = ap.load_project_metadata()
+        for pname in ["AB", "#CDE"]:
+            self.assertTrue(pname in [p["Project"] for p in ap_project_metadata],
+                            f"'{pname}' not in project metadata")
+
+    def test_update_project_metadata_add_unlisted_projects(self):
+        """
+        update: add unlisted project directories to 'projects.info'
+        """
+        # Metadata for projects
+        project_metadata = {
+            "AB": {
+                "User": "Alan Bailey",
+                "PI": "Archie Ballard",
+                "Library": "RNA-seq",
+                "Organism": "Human"
+            },
+            "CDE": {
+                "User": "Charles Edwards",
+                "PI": "Christian Eggars",
+                "Library": "ChIP-seq",
+                "Organism": "Mouse"
+            }
+        }
+        # Make an auto-process directory with projects
+        mockdir = MockAnalysisDirFactory.bcl2fastq2(
+            '231021_A00879_0087_000000000-AGEW9',
+            'novaseq',
+            project_metadata=project_metadata,
+            metadata={ "run_number": 87,
+                       "source": "local" },
+            top_dir=self.dirn)
+        mockdir.create()
+        # Remove initial entry for CDE in projects.info
+        projects_info_contents = []
+        with open(os.path.join(mockdir.dirn,"projects.info"),'rt') as fp:
+            for line in fp:
+                if not line.startswith("CDE"):
+                    projects_info_contents.append(line)
+        with open(os.path.join(mockdir.dirn,"projects.info"),'wt') as fp:
+            fp.write("".join(projects_info_contents))
+        # Set up AutoProcess instance and do the update
+        ap = AutoProcess(mockdir.dirn)
+        update(ap,
+               update_paths=False,
+               update_sync_projects=True,
+               update_project_metadata=False,
+               update_qc_reports=False)
+        # Check contents of projects.info
+        with open(os.path.join(mockdir.dirn,"projects.info"),'rt') as fp:
+            expected_lines = [
+                "#Project\tSamples\tUser\tLibrary\tSC_Platform\tOrganism\tPI\tComments",
+                "AB\tAB1,AB2\tAlan Bailey\tRNA-seq\t.\tHuman\tArchie Ballard\t.",
+                "CDE\tCDE3,CDE4\tCharles Edwards\tChIP-seq\t.\tMouse\tChristian Eggars\t."
+            ]
+            for expected, actual in zip(expected_lines, fp.read().split("\n")):
+                self.assertEqual(expected.strip(), actual.strip())
+
+    def test_update_project_metadata_add_unlisted_projects_ignore_special_names(self):
+        """
+        update: ignore unlisted project directories (don't add to 'projects.info')
+        """
+        # Metadata for projects
+        project_metadata = {
+            "AB": {
+                "User": "Alan Bailey",
+                "PI": "Archie Ballard",
+                "Library": "RNA-seq",
+                "Organism": "Human"
+            },
+            "CDE.bak": {
+                "User": "Charles Edwards",
+                "PI": "Christian Eggars",
+                "Library": "ChIP-seq",
+                "Organism": "Mouse"
+            }
+        }
+        # Make an auto-process directory with projects
+        mockdir = MockAnalysisDirFactory.bcl2fastq2(
+            '231021_A00879_0087_000000000-AGEW9',
+            'novaseq',
+            project_metadata=project_metadata,
+            metadata={ "run_number": 87,
+                       "source": "local" },
+            top_dir=self.dirn)
+        mockdir.create()
+        # Remove entry for CDE.bak in projects.info
+        projects_info_contents = []
+        with open(os.path.join(mockdir.dirn,"projects.info"),'rt') as fp:
+            for line in fp:
+                if not line.startswith("CDE.bak"):
+                    projects_info_contents.append(line)
+        with open(os.path.join(mockdir.dirn,"projects.info"),'wt') as fp:
+            fp.write("".join(projects_info_contents))
+        # Set up AutoProcess instance and do the update
+        ap = AutoProcess(mockdir.dirn)
+        update(ap,
+               update_paths=False,
+               update_sync_projects=True,
+               update_project_metadata=False,
+               update_qc_reports=False)
+        # Check contents of projects.info
+        with open(os.path.join(mockdir.dirn,"projects.info"),'rt') as fp:
+            expected_lines = [
+                "#Project\tSamples\tUser\tLibrary\tSC_Platform\tOrganism\tPI\tComments",
+                "AB\tAB1,AB2\tAlan Bailey\tRNA-seq\t.\tHuman\tArchie Ballard\t."
+            ]
+            for expected, actual in zip(expected_lines, fp.read().split("\n")):
+                self.assertEqual(expected.strip(), actual.strip())
+
     def test_update_regenerate_qc_reports(self):
         """
         update: regenerate QC reports with stale metadata
@@ -400,7 +590,11 @@ def test_update_regenerate_qc_reports(self):
             self.assertTrue(metadata_mtime <
                             os.path.getmtime(metadata_file))
         # Re-do update and check modification times
-        update(ap)
+        update(ap,
+               update_paths=False,
+               update_sync_projects=False,
+               update_project_metadata=False,
+               update_qc_reports=True)
         for project_name in project_list:
             self.assertTrue(qc_report_mtimes[project_name] <
                             os.path.getmtime(