Skip to content

Commit 4c823c5

Browse files
pjbriggspjbriggs
authored
Merge pull request #1177 from fls-bioinformatics-core/fix-archive-from-pending-to-final
'archive' command: fix bugs setting group & logging when archiving to 'final' from 'staging'
2 parents d0c409e + 2eb35bb commit 4c823c5

2 files changed

Lines changed: 197 additions & 9 deletions

File tree

‎auto_process_ngs/commands/archive_cmd.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,13 @@ def archive(ap,archive_dir=None,platform=None,year=None,
207207
extra_bcl2fastq_dirs.append(dirn)
208208
except Exception:
209209
pass
210+
# Set up runners
211+
if runner is None:
212+
rsync_runner = ap.settings.runners.rsync
213+
default_runner = ap.settings.general.default_runner
214+
else:
215+
rsync_runner = runner
216+
default_runner = runner
210217
if not is_staging:
211218
# Are there any projects to archive?
212219
try:
@@ -284,13 +291,6 @@ def archive(ap,archive_dir=None,platform=None,year=None,
284291
if dry_run:
285292
log_dir += '_dry_run'
286293
ap.set_log_dir(ap.get_log_subdir(log_dir))
287-
# Set up runners
288-
if runner is None:
289-
rsync_runner = ap.settings.runners.rsync
290-
default_runner = ap.settings.general.default_runner
291-
else:
292-
rsync_runner = runner
293-
default_runner = runner
294294
# Set log directory
295295
for r in (rsync_runner,
296296
default_runner,):
@@ -488,14 +488,14 @@ def archive(ap,archive_dir=None,platform=None,year=None,
488488
# Add to logging file
489489
if logging_file is not None:
490490
print(f"Adding details to logging file: {logging_file}")
491-
run_details = report_concise(ap)
491+
run_details = report_concise(AutoProcess(os.path.join(archive_dir, final_dest)))
492492
log_data_cmd = Command("log_seq_data.sh",
493493
logging_file,
494494
"-u",
495495
os.path.join(archive_dir,final_dest),
496496
run_details)
497+
print(f"Running {log_data_cmd}")
497498
if not dry_run:
498-
print(f"Running {log_data_cmd}")
499499
status = log_data_cmd.run_subprocess()
500500
if status != 0:
501501
logger.warning(f"Logging run to {logging_file} failed "

‎auto_process_ngs/test/commands/test_archive_cmd.py‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -671,6 +671,194 @@ def test_archive_staging_to_final(self):
671671
self.assertEqual(archived_ap.metadata.run_reference_id,
672672
"MISEQ_170901#87")
673673

674+
def test_archive_staging_to_final_set_group(self):
675+
"""
676+
archive: test archiving directly from staging dir (set group)
677+
"""
678+
# Get group for current user
679+
group_name = grp.getgrgid(os.getgid()).gr_name
680+
# Make a mock auto-process directory
681+
mockdir = MockAnalysisDirFactory.bcl2fastq2(
682+
'170901_M00879_0087_000000000-AGEW9',
683+
'miseq',
684+
metadata={ "instrument_datestamp": "170901" },
685+
top_dir=self.dirn)
686+
mockdir.create()
687+
# Make a mock archive directory
688+
archive_dir = os.path.join(self.dirn,"archive")
689+
final_dir = os.path.join(archive_dir,
690+
"2017",
691+
"miseq")
692+
os.makedirs(final_dir)
693+
self.assertTrue(os.path.isdir(final_dir))
694+
self.assertEqual(len(os.listdir(final_dir)),0)
695+
# Make autoprocess instance and set required metadata
696+
ap = AutoProcess(analysis_dir=mockdir.dirn,
697+
settings=self.settings)
698+
ap.set_metadata("source","testing")
699+
ap.set_metadata("run_number","87")
700+
ap.save_metadata()
701+
# Move to the archive area as a "pending" directory
702+
os.rename(mockdir.dirn,
703+
os.path.join(
704+
archive_dir,
705+
"2017",
706+
"miseq",
707+
"__170901_M00879_0087_000000000-AGEW9_analysis.pending"))
708+
# Load pending dir into a new autoprocess instance
709+
ap = AutoProcess(
710+
analysis_dir=os.path.join(
711+
archive_dir,
712+
"2017",
713+
"miseq",
714+
"__170901_M00879_0087_000000000-AGEW9_analysis.pending"))
715+
# Staging archiving attempt should fail
716+
self.assertRaises(Exception,
717+
archive,
718+
ap,
719+
archive_dir=archive_dir,
720+
year='2017',platform='miseq',
721+
read_only_fastqs=False,
722+
group=group_name,
723+
logging_file=None,
724+
final=False)
725+
staging_dir = os.path.join(
726+
final_dir,
727+
"__170901_M00879_0087_000000000-AGEW9_analysis.pending")
728+
final_archive_dir = os.path.join(
729+
final_dir,
730+
"170901_M00879_0087_000000000-AGEW9_analysis")
731+
self.assertFalse(os.path.exists(final_archive_dir))
732+
self.assertEqual(len(os.listdir(final_dir)),1)
733+
# Copy to final should work
734+
status = archive(ap,
735+
archive_dir=archive_dir,
736+
year='2017',platform='miseq',
737+
read_only_fastqs=False,
738+
group=group_name,
739+
logging_file=None,
740+
final=True)
741+
self.assertEqual(status,0)
742+
self.assertTrue(os.path.exists(final_archive_dir))
743+
self.assertFalse(os.path.exists(staging_dir))
744+
self.assertEqual(len(os.listdir(final_dir)),1)
745+
# Check contents
746+
dirs = ("AB","CDE","logs","undetermined")
747+
for d in dirs:
748+
d = os.path.join(final_archive_dir,d)
749+
self.assertTrue(os.path.exists(d))
750+
files = ("auto_process.info",
751+
"custom_SampleSheet.csv",
752+
"metadata.info",
753+
"projects.info",
754+
"SampleSheet.orig.csv")
755+
for f in files:
756+
f = os.path.join(final_archive_dir,f)
757+
self.assertTrue(os.path.exists(f))
758+
# Check paths are updated
759+
archived_ap = AutoProcess(analysis_dir=final_archive_dir,
760+
settings=self.settings)
761+
self.assertEqual(archived_ap.params.analysis_dir,
762+
final_archive_dir)
763+
# Check run ID and reference
764+
self.assertEqual(archived_ap.metadata.run_id,
765+
"MISEQ_170901#87")
766+
self.assertEqual(archived_ap.metadata.run_reference_id,
767+
"MISEQ_170901#87")
768+
769+
def test_archive_staging_to_final_with_logging_file(self):
770+
"""
771+
archive: test archiving directly from staging dir (with logging file)
772+
"""
773+
# Make a mock auto-process directory
774+
mockdir = MockAnalysisDirFactory.bcl2fastq2(
775+
'170901_M00879_0087_000000000-AGEW9',
776+
'miseq',
777+
metadata={ "instrument_datestamp": "170901" },
778+
top_dir=self.dirn)
779+
mockdir.create()
780+
# Make a mock archive directory
781+
archive_dir = os.path.join(self.dirn,"archive")
782+
final_dir = os.path.join(archive_dir,
783+
"2017",
784+
"miseq")
785+
os.makedirs(final_dir)
786+
self.assertTrue(os.path.isdir(final_dir))
787+
self.assertEqual(len(os.listdir(final_dir)),0)
788+
# Make autoprocess instance and set required metadata
789+
ap = AutoProcess(analysis_dir=mockdir.dirn,
790+
settings=self.settings)
791+
ap.set_metadata("source","testing")
792+
ap.set_metadata("run_number","87")
793+
ap.save_metadata()
794+
# Logging file path
795+
logging_file = os.path.join(self.dirn, "SEQ_DATA.log")
796+
# Move to the archive area as a "pending" directory
797+
os.rename(mockdir.dirn,
798+
os.path.join(
799+
archive_dir,
800+
"2017",
801+
"miseq",
802+
"__170901_M00879_0087_000000000-AGEW9_analysis.pending"))
803+
# Load pending dir into a new autoprocess instance
804+
ap = AutoProcess(
805+
analysis_dir=os.path.join(
806+
archive_dir,
807+
"2017",
808+
"miseq",
809+
"__170901_M00879_0087_000000000-AGEW9_analysis.pending"))
810+
# Staging archiving attempt should fail
811+
self.assertRaises(Exception,
812+
archive,
813+
ap,
814+
archive_dir=archive_dir,
815+
year='2017',platform='miseq',
816+
read_only_fastqs=False,
817+
logging_file=logging_file,
818+
final=False)
819+
staging_dir = os.path.join(
820+
final_dir,
821+
"__170901_M00879_0087_000000000-AGEW9_analysis.pending")
822+
final_archive_dir = os.path.join(
823+
final_dir,
824+
"170901_M00879_0087_000000000-AGEW9_analysis")
825+
self.assertFalse(os.path.exists(final_archive_dir))
826+
self.assertEqual(len(os.listdir(final_dir)),1)
827+
# Copy to final should work
828+
status = archive(ap,
829+
archive_dir=archive_dir,
830+
year='2017',platform='miseq',
831+
read_only_fastqs=False,
832+
logging_file=logging_file,
833+
final=True)
834+
self.assertEqual(status,0)
835+
self.assertTrue(os.path.exists(final_archive_dir))
836+
self.assertFalse(os.path.exists(staging_dir))
837+
self.assertEqual(len(os.listdir(final_dir)),1)
838+
# Check contents
839+
dirs = ("AB","CDE","logs","undetermined")
840+
for d in dirs:
841+
d = os.path.join(final_archive_dir,d)
842+
self.assertTrue(os.path.exists(d))
843+
files = ("auto_process.info",
844+
"custom_SampleSheet.csv",
845+
"metadata.info",
846+
"projects.info",
847+
"SampleSheet.orig.csv")
848+
for f in files:
849+
f = os.path.join(final_archive_dir,f)
850+
self.assertTrue(os.path.exists(f))
851+
# Check paths are updated
852+
archived_ap = AutoProcess(analysis_dir=final_archive_dir,
853+
settings=self.settings)
854+
self.assertEqual(archived_ap.params.analysis_dir,
855+
final_archive_dir)
856+
# Check run ID and reference
857+
self.assertEqual(archived_ap.metadata.run_id,
858+
"MISEQ_170901#87")
859+
self.assertEqual(archived_ap.metadata.run_reference_id,
860+
"MISEQ_170901#87")
861+
674862
def test_archive_automatically_sets_correct_year(self):
675863
"""archive: test archiving sets the year correctly if not specified
676864
"""

0 commit comments

Comments
 (0)