Skip to content

'archive' command: fix bugs setting group & logging when archiving to 'final' from 'staging' #1177

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pjbriggs merged 3 commits into from
Jun 30, 2026
Merged

'archive' command: fix bugs setting group & logging when archiving to 'final' from 'staging' #1177

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a14797e
'archive' command: fix bug when archiving to final from staging.
pjbriggs Jun 30, 2026
c6f4e26
'archive' command: add extra tests for archiving directly from 'stagi…
pjbriggs Jun 30, 2026
2eb35bb
'archive': fix bug with logging file for 'staging' to 'final' archive.
pjbriggs Jun 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions auto_process_ngs/commands/archive_cmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,13 @@ def archive(ap,archive_dir=None,platform=None,year=None,
extra_bcl2fastq_dirs.append(dirn)
except Exception:
pass
# Set up runners
if runner is None:
rsync_runner = ap.settings.runners.rsync
default_runner = ap.settings.general.default_runner
else:
rsync_runner = runner
default_runner = runner
if not is_staging:
# Are there any projects to archive?
try:
Expand Down Expand Up @@ -284,13 +291,6 @@ def archive(ap,archive_dir=None,platform=None,year=None,
if dry_run:
log_dir += '_dry_run'
ap.set_log_dir(ap.get_log_subdir(log_dir))
# Set up runners
if runner is None:
rsync_runner = ap.settings.runners.rsync
default_runner = ap.settings.general.default_runner
else:
rsync_runner = runner
default_runner = runner
# Set log directory
for r in (rsync_runner,
default_runner,):
Expand Down Expand Up @@ -488,14 +488,14 @@ def archive(ap,archive_dir=None,platform=None,year=None,
# Add to logging file
if logging_file is not None:
print(f"Adding details to logging file: {logging_file}")
run_details = report_concise(ap)
run_details = report_concise(AutoProcess(os.path.join(archive_dir, final_dest)))
log_data_cmd = Command("log_seq_data.sh",
logging_file,
"-u",
os.path.join(archive_dir,final_dest),
run_details)
print(f"Running {log_data_cmd}")
if not dry_run:
print(f"Running {log_data_cmd}")
status = log_data_cmd.run_subprocess()
if status != 0:
logger.warning(f"Logging run to {logging_file} failed "
Expand Down
188 changes: 188 additions & 0 deletions auto_process_ngs/test/commands/test_archive_cmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -671,6 +671,194 @@ def test_archive_staging_to_final(self):
self.assertEqual(archived_ap.metadata.run_reference_id,
"MISEQ_170901#87")

def test_archive_staging_to_final_set_group(self):
"""
archive: test archiving directly from staging dir (set group)
"""
# Get group for current user
group_name = grp.getgrgid(os.getgid()).gr_name
# Make a mock auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'170901_M00879_0087_000000000-AGEW9',
'miseq',
metadata={ "instrument_datestamp": "170901" },
top_dir=self.dirn)
mockdir.create()
# Make a mock archive directory
archive_dir = os.path.join(self.dirn,"archive")
final_dir = os.path.join(archive_dir,
"2017",
"miseq")
os.makedirs(final_dir)
self.assertTrue(os.path.isdir(final_dir))
self.assertEqual(len(os.listdir(final_dir)),0)
# Make autoprocess instance and set required metadata
ap = AutoProcess(analysis_dir=mockdir.dirn,
settings=self.settings)
ap.set_metadata("source","testing")
ap.set_metadata("run_number","87")
ap.save_metadata()
# Move to the archive area as a "pending" directory
os.rename(mockdir.dirn,
os.path.join(
archive_dir,
"2017",
"miseq",
"__170901_M00879_0087_000000000-AGEW9_analysis.pending"))
# Load pending dir into a new autoprocess instance
ap = AutoProcess(
analysis_dir=os.path.join(
archive_dir,
"2017",
"miseq",
"__170901_M00879_0087_000000000-AGEW9_analysis.pending"))
# Staging archiving attempt should fail
self.assertRaises(Exception,
archive,
ap,
archive_dir=archive_dir,
year='2017',platform='miseq',
read_only_fastqs=False,
group=group_name,
logging_file=None,
final=False)
staging_dir = os.path.join(
final_dir,
"__170901_M00879_0087_000000000-AGEW9_analysis.pending")
final_archive_dir = os.path.join(
final_dir,
"170901_M00879_0087_000000000-AGEW9_analysis")
self.assertFalse(os.path.exists(final_archive_dir))
self.assertEqual(len(os.listdir(final_dir)),1)
# Copy to final should work
status = archive(ap,
archive_dir=archive_dir,
year='2017',platform='miseq',
read_only_fastqs=False,
group=group_name,
logging_file=None,
final=True)
self.assertEqual(status,0)
self.assertTrue(os.path.exists(final_archive_dir))
self.assertFalse(os.path.exists(staging_dir))
self.assertEqual(len(os.listdir(final_dir)),1)
# Check contents
dirs = ("AB","CDE","logs","undetermined")
for d in dirs:
d = os.path.join(final_archive_dir,d)
self.assertTrue(os.path.exists(d))
files = ("auto_process.info",
"custom_SampleSheet.csv",
"metadata.info",
"projects.info",
"SampleSheet.orig.csv")
for f in files:
f = os.path.join(final_archive_dir,f)
self.assertTrue(os.path.exists(f))
# Check paths are updated
archived_ap = AutoProcess(analysis_dir=final_archive_dir,
settings=self.settings)
self.assertEqual(archived_ap.params.analysis_dir,
final_archive_dir)
# Check run ID and reference
self.assertEqual(archived_ap.metadata.run_id,
"MISEQ_170901#87")
self.assertEqual(archived_ap.metadata.run_reference_id,
"MISEQ_170901#87")

def test_archive_staging_to_final_with_logging_file(self):
"""
archive: test archiving directly from staging dir (with logging file)
"""
# Make a mock auto-process directory
mockdir = MockAnalysisDirFactory.bcl2fastq2(
'170901_M00879_0087_000000000-AGEW9',
'miseq',
metadata={ "instrument_datestamp": "170901" },
top_dir=self.dirn)
mockdir.create()
# Make a mock archive directory
archive_dir = os.path.join(self.dirn,"archive")
final_dir = os.path.join(archive_dir,
"2017",
"miseq")
os.makedirs(final_dir)
self.assertTrue(os.path.isdir(final_dir))
self.assertEqual(len(os.listdir(final_dir)),0)
# Make autoprocess instance and set required metadata
ap = AutoProcess(analysis_dir=mockdir.dirn,
settings=self.settings)
ap.set_metadata("source","testing")
ap.set_metadata("run_number","87")
ap.save_metadata()
# Logging file path
logging_file = os.path.join(self.dirn, "SEQ_DATA.log")
# Move to the archive area as a "pending" directory
os.rename(mockdir.dirn,
os.path.join(
archive_dir,
"2017",
"miseq",
"__170901_M00879_0087_000000000-AGEW9_analysis.pending"))
# Load pending dir into a new autoprocess instance
ap = AutoProcess(
analysis_dir=os.path.join(
archive_dir,
"2017",
"miseq",
"__170901_M00879_0087_000000000-AGEW9_analysis.pending"))
# Staging archiving attempt should fail
self.assertRaises(Exception,
archive,
ap,
archive_dir=archive_dir,
year='2017',platform='miseq',
read_only_fastqs=False,
logging_file=logging_file,
final=False)
staging_dir = os.path.join(
final_dir,
"__170901_M00879_0087_000000000-AGEW9_analysis.pending")
final_archive_dir = os.path.join(
final_dir,
"170901_M00879_0087_000000000-AGEW9_analysis")
self.assertFalse(os.path.exists(final_archive_dir))
self.assertEqual(len(os.listdir(final_dir)),1)
# Copy to final should work
status = archive(ap,
archive_dir=archive_dir,
year='2017',platform='miseq',
read_only_fastqs=False,
logging_file=logging_file,
final=True)
self.assertEqual(status,0)
self.assertTrue(os.path.exists(final_archive_dir))
self.assertFalse(os.path.exists(staging_dir))
self.assertEqual(len(os.listdir(final_dir)),1)
# Check contents
dirs = ("AB","CDE","logs","undetermined")
for d in dirs:
d = os.path.join(final_archive_dir,d)
self.assertTrue(os.path.exists(d))
files = ("auto_process.info",
"custom_SampleSheet.csv",
"metadata.info",
"projects.info",
"SampleSheet.orig.csv")
for f in files:
f = os.path.join(final_archive_dir,f)
self.assertTrue(os.path.exists(f))
# Check paths are updated
archived_ap = AutoProcess(analysis_dir=final_archive_dir,
settings=self.settings)
self.assertEqual(archived_ap.params.analysis_dir,
final_archive_dir)
# Check run ID and reference
self.assertEqual(archived_ap.metadata.run_id,
"MISEQ_170901#87")
self.assertEqual(archived_ap.metadata.run_reference_id,
"MISEQ_170901#87")

def test_archive_automatically_sets_correct_year(self):
"""archive: test archiving sets the year correctly if not specified
"""
Expand Down
Loading