add pacbio_absquant_v12 to require sample mass (#366)

* add pacbio_absquant_v12 to require sample mass

* clarify inheritance comment

Author: AmandaBirmingham

metapool/abs_quant_sample_info.py

@@ -1,8 +1,10 @@
 import os
 import pandas
+from typing import Optional
 import yaml
 
-from typing import Optional
+from metapool.mp_strings import CALC_MASS_SAMPLE_ALIQUOT_INPUT_G_KEY
+
 
 # Relative path to default config file
 DEFAULT_CONFIG_FP = 'data/abs_quant_sample_info_calc.yml'
@@ -46,7 +48,6 @@
 CALC_DENSITY_HOMOGENATE_G_ML_KEY = 'calc_density_homogenate_g_ml'
 CALC_MASS_HOMOGENATE_ALIQUOT_INPUT_G_KEY = \
     'calc_mass_homogenate_aliquot_input_g'
-CALC_MASS_SAMPLE_ALIQUOT_INPUT_G_KEY = 'calc_mass_sample_aliquot_input_g'
 CALC_MASS_SAMPLE_ALIQUOT_INPUT_MG_KEY = 'calc_mass_sample_aliquot_input_mg'
 CALC_MASS_STORAGE_LIQUID_ALIQUOT_INPUT_G = \
     'calc_mass_storage_liquid_aliquot_input_g'

metapool/mp_strings.py

@@ -70,6 +70,7 @@
 BARCODE_ID_KEY = 'barcode_id'
 TWIST_ADAPTOR_ID_KEY = 'twist_adaptor_id'
 SYNDNA_IS_TWISTED_KEY = 'syndna_is_twisted'
+CALC_MASS_SAMPLE_ALIQUOT_INPUT_G_KEY = 'calc_mass_sample_aliquot_input_g'
 
 
 def parse_project_name(project_name):

metapool/sample_sheet.py

@@ -16,7 +16,8 @@
     PROJECT_FULL_NAME_KEY, TUBECODE_KEY, SYNDNA_POOL_MASS_NG_KEY, \
     SYNDNA_POOL_NUM_KEY, ELUTION_VOL_KEY, EXTRACTED_GDNA_CONC_KEY, \
     LIB_CONSTRUCT_PROTOCOL_KEY, PM_WELL_ID_384_KEY, DESTINATION_WELL_384_KEY, \
-    BARCODE_ID_KEY, TWIST_ADAPTOR_ID_KEY, SYNDNA_IS_TWISTED_KEY
+    BARCODE_ID_KEY, TWIST_ADAPTOR_ID_KEY, SYNDNA_IS_TWISTED_KEY, \
+    CALC_MASS_SAMPLE_ALIQUOT_INPUT_G_KEY
 from metapool.util import convert_to_bool
 from metapool.metapool import (bcl_scrub_name, sequencer_i5_index)
 from metapool.sequencers import is_i5_revcomp_sequencer, get_sequencer_type, \
@@ -1542,7 +1543,7 @@ def __init__(self, path=None, defer_validate=False):
         self._validate_on_load(path, defer_validate)
 
 
-class AbsQuantMixin(object):
+class AbsQuantMixinV1(object):
     _ABSQUANT_SPECIFIC_COLUMNS = (
         SYNDNA_POOL_MASS_NG_KEY, EXTRACTED_GDNA_CONC_KEY,
         ELUTION_VOL_KEY, SYNDNA_POOL_NUM_KEY)
@@ -1555,7 +1556,35 @@ class AbsQuantMixin(object):
     })
 
     # Reminder: the method of adding carried prep columns and data columns used
-    # below ONLY works if the mixin is the last class in the inheritance list
+    # below ONLY works if the mixin is the leaf class in the inheritance
+    # (which, in python, is the *first* one listed in the class definition)
+    # and if the class that inherits from it does not explicitly set these
+    # attributes in its own __init__ method.  Safest not to set these
+    # attributes in leaf classes at all (specific sheet versions) for leaf
+    # class that uses a mixin like this.
+    def __init__(self, path=None, defer_validate=False):
+        super().__init__(path=path, defer_validate=True)
+        self._remapper = self._extend_mapping_type(self._ABSQUANT_REMAPPER)
+        self._data_columns = \
+            self._data_columns + self._ABSQUANT_SPECIFIC_COLUMNS
+        if self._CARRIED_PREP_COLUMNS is not None:
+            self._CARRIED_PREP_COLUMNS = \
+                self._CARRIED_PREP_COLUMNS + self._ABSQUANT_SPECIFIC_COLUMNS
+        self._validate_on_load(path, defer_validate)
+
+
+class AbsQuantMixinV2(object):
+    _ABSQUANT_SPECIFIC_COLUMNS = AbsQuantMixinV1._ABSQUANT_SPECIFIC_COLUMNS + \
+        (CALC_MASS_SAMPLE_ALIQUOT_INPUT_G_KEY,)
+    _ABSQUANT_REMAPPER = MappingProxyType(
+        AbsQuantMixinV1._ABSQUANT_REMAPPER |
+        {CALC_MASS_SAMPLE_ALIQUOT_INPUT_G_KEY:
+         CALC_MASS_SAMPLE_ALIQUOT_INPUT_G_KEY}
+    )
+
+    # Reminder: the method of adding carried prep columns and data columns used
+    # below ONLY works if the mixin is the leaf class in the inheritance
+    # (which, in python, is the *first* one listed in the class definition)
     # and if the class that inherits from it does not explicitly set these
     # attributes in its own __init__ method.  Safest not to set these
     # attributes in leaf classes at all (specific sheet versions) for leaf
@@ -1680,7 +1709,8 @@ class TellseqMetagSampleSheetv10(KLTellSeqSampleSheet):
     _HEADER[_ASSAY_KEY] = _METAGENOMIC
 
 
-class TellseqAbsquantMetagSampleSheetv10(AbsQuantMixin, KLTellSeqSampleSheet):
+class TellseqAbsquantMetagSampleSheetv10(
+        AbsQuantMixinV1, KLTellSeqSampleSheet):
     _HEADER = KLSampleSheet._HEADER.copy()
     _HEADER[_SHEET_TYPE_KEY] = TELLSEQ_ABSQUANT_SHEET_TYPE
     _HEADER[_SHEET_VERSION_KEY] = '10'
@@ -1828,7 +1858,7 @@ class PacBioMetagSampleSheetv10(PacBioSampleSheet):
     _HEADER[_ASSAY_KEY] = _METAGENOMIC
 
 
-class PacBioAbsquantSampleSheetv10(AbsQuantMixin, PacBioSampleSheet):
+class PacBioAbsquantSampleSheetv10(AbsQuantMixinV1, PacBioSampleSheet):
     _HEADER = PacBioSampleSheet._HEADER.copy()
     _HEADER[_SHEET_TYPE_KEY] = PACBIO_ABSQUANT_SHEET_TYPE
     _HEADER[_SHEET_VERSION_KEY] = '10'
@@ -1886,15 +1916,42 @@ class PacBioMetagSampleSheetv11(PacBioSampleSheetWithTwistAdapters):
     _HEADER[_ASSAY_KEY] = _METAGENOMIC
 
 
-class TwistAbsquantMixin(AbsQuantMixin):
+class TwistAbsquantMixinV1(AbsQuantMixinV1):
+    _TWIST_ABSQUANT_SPECIFIC_COLUMNS = (SYNDNA_IS_TWISTED_KEY,)
+
+    _TWIST_ABSQUANT_REMAPPER = MappingProxyType({
+        SYNDNA_IS_TWISTED_KEY: SYNDNA_IS_TWISTED_KEY,
+    })
+
+    # Reminder: the method of adding carried prep columns and data columns used
+    # below ONLY works if the mixin is the leaf class in the inheritance
+    # (which, in python, is the *first* one listed in the class definition)
+    # and if the class that inherits from it does not explicitly set these
+    # attributes in its own __init__ method.  Safest not to set these
+    # attributes in leaf classes at all (specific sheet versions) for leaf
+    # class that uses a mixin like this.
+    def __init__(self, path=None, defer_validate=False):
+        super().__init__(path=path, defer_validate=True)
+        self._remapper = self._extend_mapping_type(
+            self._TWIST_ABSQUANT_REMAPPER)
+        self._data_columns = self._data_columns + \
+            self._TWIST_ABSQUANT_SPECIFIC_COLUMNS
+        if self._CARRIED_PREP_COLUMNS is not None:
+            self._CARRIED_PREP_COLUMNS = self._CARRIED_PREP_COLUMNS + \
+                self._TWIST_ABSQUANT_SPECIFIC_COLUMNS
+        self._validate_on_load(path, defer_validate)
+
+
+class TwistAbsquantMixinV2(AbsQuantMixinV2):
     _TWIST_ABSQUANT_SPECIFIC_COLUMNS = (SYNDNA_IS_TWISTED_KEY,)
 
     _TWIST_ABSQUANT_REMAPPER = MappingProxyType({
         SYNDNA_IS_TWISTED_KEY: SYNDNA_IS_TWISTED_KEY,
     })
 
     # Reminder: the method of adding carried prep columns and data columns used
-    # below ONLY works if the mixin is the last class in the inheritance list
+    # below ONLY works if the mixin is the leaf class in the inheritance
+    # (which, in python, is the *first* one listed in the class definition)
     # and if the class that inherits from it does not explicitly set these
     # attributes in its own __init__ method.  Safest not to set these
     # attributes in leaf classes at all (specific sheet versions) for leaf
@@ -1912,13 +1969,21 @@ def __init__(self, path=None, defer_validate=False):
 
 
 class PacBioAbsquantSampleSheetv11(
-        TwistAbsquantMixin, PacBioSampleSheetWithTwistAdapters):
+        TwistAbsquantMixinV1, PacBioSampleSheetWithTwistAdapters):
     _HEADER = PacBioSampleSheetWithTwistAdapters._HEADER.copy()
     _HEADER[_SHEET_TYPE_KEY] = PACBIO_ABSQUANT_SHEET_TYPE
     _HEADER[_SHEET_VERSION_KEY] = '11'
     _HEADER[_ASSAY_KEY] = _METAGENOMIC
 
 
+class PacBioAbsquantSampleSheetv12(
+        TwistAbsquantMixinV2, PacBioSampleSheetWithTwistAdapters):
+    _HEADER = PacBioSampleSheetWithTwistAdapters._HEADER.copy()
+    _HEADER[_SHEET_TYPE_KEY] = PACBIO_ABSQUANT_SHEET_TYPE
+    _HEADER[_SHEET_VERSION_KEY] = '12'
+    _HEADER[_ASSAY_KEY] = _METAGENOMIC
+
+
 class MetagenomicSampleSheetv102(
         KatharoseqMixin, KLSampleSheetWithSampleContext):
     # Adds support for optional KATHAROSEQ columns in [Data] section.
@@ -2008,7 +2073,7 @@ def __init__(self, path=None, defer_validate=False):
         self._validate_on_load(path, defer_validate)
 
 
-class AbsQuantSampleSheetv10(AbsQuantMixin, KLSampleSheetWithReplicates):
+class AbsQuantSampleSheetv10(AbsQuantMixinV1, KLSampleSheetWithReplicates):
     _HEADER = {
         'IEMFileVersion': '4',
         _SHEET_TYPE_KEY: ABSQUANT_SHEET_TYPE,
@@ -2024,7 +2089,7 @@ class AbsQuantSampleSheetv10(AbsQuantMixin, KLSampleSheetWithReplicates):
     }
 
 
-class AbsQuantSampleSheetv11(AbsQuantMixin, KLSampleSheetWithSampleContext):
+class AbsQuantSampleSheetv11(AbsQuantMixinV1, KLSampleSheetWithSampleContext):
     _HEADER = AbsQuantSampleSheetv10._HEADER.copy()
     _HEADER[_SHEET_TYPE_KEY] = ABSQUANT_SHEET_TYPE
     _HEADER[_SHEET_VERSION_KEY] = '11'
@@ -2243,6 +2308,7 @@ def _id_sample_sheet_class(sheet_type, sheet_version, assay_type):
             _METAGENOMIC: {
                 '10': PacBioAbsquantSampleSheetv10,
                 '11': PacBioAbsquantSampleSheetv11,
+                '12': PacBioAbsquantSampleSheetv12,
             },
         }
     }

metapool/tests/data/good_pacbio_absquantv12.csv

@@ -0,0 +1,27 @@
+[Header],,,,,,,,,,,,,,
+SheetType,pacbio_absquant,,,,,,,,,,,,,
+SheetVersion,12,,,,,,,,,,,,,
+Investigator Name,Knight,,,,,,,,,,,,,
+Experiment Name,RKL_experiment,,,,,,,,,,,,,
+Date,2025-08-29,,,,,,,,,,,,,
+Assay,Metagenomic,,,,,,,,,,,,,
+Description,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,
+[Data],,,,,,,,,,,,,,
+Sample_ID,Sample_Name,Sample_Plate,Sample_Well,barcode_id,twist_adaptor_id,Sample_Project,Well_description,mass_syndna_input_ng,extracted_gdna_concentration_ng_ul,vol_extracted_elution_ul,syndna_pool_number,calc_mass_sample_aliquot_input_g,syndna_is_twisted,Lane
+sample_1,sample.1,sample_plate_1,A1,bc3011,16_UDI_1_A01_F--16_UDI_1_A01_R,MyProject_99999,sample_plate_1.sample.1.A1,0.2,1,1.1,1,0.02,False,1
+sample_2,sample.2,sample_plate_1,A2,bc0112,16_UDI_2_B01_F--16_UDI_2_B01_R,MyProject_99999,sample_plate_1.sample.2.A2,0.22,1,1.1,1,0.012,False,1
+sample_3,sample.3,sample_plate_1,A3,bc9992,16_UDI_5_E01_F--16_UDI_5_E01_R,MyProject_99999,sample_plate_1.sample.3.A3,0.25,1,1.1,1,0.105,False,1
+,,,,,,,,,,,,,,
+[Bioinformatics],,,,,,,,,,,,,,
+Sample_Project,QiitaID,HumanFiltering,library_construction_protocol,experiment_design_description,contains_replicates,,,,,,,,,
+MyProject_99999,99999,False,some protocol,some description,False,,,,,,,,,
+,,,,,,,,,,,,,,
+[Contact],,,,,,,,,,,,,,
+Sample_Project,Email,,,,,,,,,,,,,
+MyProject_99999,foo@bar.org,,,,,,,,,,,,,
+,,,,,,,,,,,,,,
+[SampleContext],,,,,,,,,,,,,,
+sample_name,sample_type,primary_qiita_study,secondary_qiita_studies,,,,,,,,,,,
+sample.3,control blank,99999,,,,,,,,,,,,
+,,,,,,,,,,,,,,

metapool/tests/test_sample_sheet.py

@@ -31,6 +31,7 @@
                                    PacBioAbsquantSampleSheetv10,
                                    PacBioMetagSampleSheetv11,
                                    PacBioAbsquantSampleSheetv11,
+                                   PacBioAbsquantSampleSheetv12,
                                    sample_sheet_to_dataframe,
                                    make_sample_sheet, load_sample_sheet,
                                    demux_sample_sheet, sheet_needs_demuxing,
@@ -3296,6 +3297,54 @@ def test_PacBioAbsquantSampleSheetv11_roundtrip(self):
         self._help_test_roundtrip_sample_sheet(self.sheet_class)
 
 
+class PacBioAbsquantSampleSheetv12CreationTests(
+        SampleSheetLoadMakeAndLoadTests):
+    sheet_class = PacBioAbsquantSampleSheetv12
+    sample_sheet_name = "good_pacbio_absquantv12.csv"
+
+    _INPUT_COLS = PacBioMetagSampleSheetv11CreationTests._INPUT_COLS.copy() + \
+        ['mass_syndna_input_ng', 'extracted_gdna_concentration_ng_ul',
+         'vol_extracted_elution_ul', 'syndna_pool_number',
+         'calc_mass_sample_aliquot_input_g', 'syndna_is_twisted']
+
+    _INPUT_DATA = [
+        ['sample_1', 'sample.1', '1', '1', 'False',
+         'A1', 'bc3011', '16_UDI_1_A01_F--16_UDI_1_A01_R',
+         'sample_plate_1', 'MyProject_99999',
+         '0.2', '1.0', '1.1', '1', '0.02', 'False'],
+        ['sample_2', 'sample.2', '2', '1', 'False',
+         'A2', 'bc0112', '16_UDI_2_B01_F--16_UDI_2_B01_R',
+         'sample_plate_1', 'MyProject_99999',
+         '0.22', '1.0', '1.1', '1', '0.012', 'False'],
+        ['sample_3', 'sample.3', '3', '1', 'False',
+         'A3', 'bc9992', '16_UDI_5_E01_F--16_UDI_5_E01_R',
+         'sample_plate_1', 'MyProject_99999',
+         '0.25', '1.0', '1.1', '1', '0.105', 'False'],
+    ]
+
+    _OUTPUT_COLS = \
+        PacBioMetagSampleSheetv11CreationTests._OUTPUT_COLS.copy() + [
+            'mass_syndna_input_ng', 'extracted_gdna_concentration_ng_ul',
+            'vol_extracted_elution_ul', 'syndna_pool_number',
+            'calc_mass_sample_aliquot_input_g', 'syndna_is_twisted']
+
+    _BIOINFORMATICS = PacBioMetagSampleSheetv11CreationTests._BIOINFORMATICS
+
+    _SAMPLE_CONTEXT = MetagenomicSampleSheetv101CreationTests._SAMPLE_CONTEXT
+
+    def test_PacBioAbsquantSampleSheetv12_instantiate_from_path(self):
+        self._help_test_instantiate_sample_sheet_from_path(self.sheet_class)
+
+    def test_PacBioAbsquantSampleSheetv12_make_sample_sheet(self):
+        self._help_test_make_sample_sheet(self.sheet_class, sequencer="Revio")
+
+    def test_PacBioAbsquantSampleSheetv12_load_sample_sheet(self):
+        self._help_test_load_sample_sheet(self.sheet_class)
+
+    def test_PacBioAbsquantSampleSheetv12_roundtrip(self):
+        self._help_test_roundtrip_sample_sheet(self.sheet_class)
+
+
 class AbsQuantSampleSheetv10CreationTests(SampleSheetLoadMakeAndLoadTests):
     sheet_class = AbsQuantSampleSheetv10
     sample_sheet_name = "good_abs_quant_metagv10.csv"

notebooks/pacbio_sample_sheet_builder.ipynb

@@ -22,7 +22,8 @@
     "from metapool.mp_strings import (\n",
     "    PM_WELL_KEY, BARCODE_ID_KEY, PM_BLANK_KEY, SYNDNA_POOL_MASS_NG_KEY,\n",
     "    ELUTION_VOL_KEY, EXTRACTED_GDNA_CONC_KEY, SYNDNA_POOL_NUM_KEY,\n",
-    "    TWIST_ADAPTOR_ID_KEY, SYNDNA_IS_TWISTED_KEY)\n",
+    "    TWIST_ADAPTOR_ID_KEY, SYNDNA_IS_TWISTED_KEY,\n",
+    "    CALC_MASS_SAMPLE_ALIQUOT_INPUT_G_KEY)\n",
     "from metapool.metapool import bcl_scrub_name\n",
     "\n",
     "%watermark -i -v -iv -m -h -p metapool,sample_sheet,openpyxl -u"
@@ -89,7 +90,8 @@
     "# run_title_key = \"Sequencing Run Name\"\n",
     "## ---------------------------------\n",
     "\n",
-    "# experimental_design_desc = \"fecal samples for metagenomic sequencing\""
+    "# experimental_design_desc = \"fecal samples for metagenomic sequencing\"\n",
+    "# contact_email = \"g@example.com\""
    ]
   },
   {
@@ -101,7 +103,8 @@
     "# DEFAULTS: DO NOT CHANGE THESE\n",
     "# unless you definitely know what you are doing\n",
     "lib_const_protocol = \"Knight Lab PacBio\"\n",
-    "sheet_type = PACBIO_METAG_SHEET_TYPE"
+    "sheet_type = PACBIO_METAG_SHEET_TYPE\n",
+    "sheet_version = '11'  # metag sheet version"
    ]
   },
   {
@@ -113,16 +116,18 @@
     "# CONSTANTS: DO NOT CHANGE THESE EITHER\n",
     "# unless you definitely know what you are doing\n",
     "ISOLATE_STR = \"isolate\"\n",
-    "SHEET_VERSION_NUM = \"11\"  # for both metag and absquant\n",
+    "ABSQUANT_SHEET_VERSION_NUM = \"12\"\n",
     "ASSUMED_SYNDNA_POOL_NUM = \"2000\"\n",
     "EXPECTED_ABSQUANT_COLS = [\n",
     "    SYNDNA_POOL_MASS_NG_KEY,    # mass_syndna_input_ng\n",
     "    ELUTION_VOL_KEY,            # vol_extracted_elution_ul\n",
     "    EXTRACTED_GDNA_CONC_KEY,    # extracted_gdna_concentration_ng_ul\n",
-    "    SYNDNA_IS_TWISTED_KEY       # syndna_is_twisted\n",
+    "    SYNDNA_IS_TWISTED_KEY,      # syndna_is_twisted\n",
+    "    CALC_MASS_SAMPLE_ALIQUOT_INPUT_G_KEY  # calc_mass_sample_aliquot_input_g\n",
     "]\n",
-    "# NB: calc_mass_sample_aliquot_input_g is needed to do the actual abs quant,\n",
-    "# but not for the sample sheet generation"
+    "\n",
+    "# NB: there are other absquant input metrics (for surface area or volume)\n",
+    "# that require OTHER input columns; these aren't yet supported here"
    ]
   },
   {
@@ -389,6 +394,7 @@
     "    lib_const_protocol += \" absquant\"\n",
     "    experimental_design_desc += \" with absquant\"\n",
     "    sheet_type = PACBIO_ABSQUANT_SHEET_TYPE\n",
+    "    sheet_version = ABSQUANT_SHEET_VERSION_NUM\n",
     "    plate_df[SYNDNA_POOL_NUM_KEY] = ASSUMED_SYNDNA_POOL_NUM\n",
     "\n",
     "    # set the SYNDNA_IS_TWISTED_KEY column to empty string for anything\n",
@@ -425,8 +431,8 @@
     "# fill the non-project-specific parts of the metadata dictionary\n",
     "metadata = {\n",
     "    _ASSAY_KEY: _METAGENOMIC,\n",
-    "    _SHEET_VERSION_KEY: SHEET_VERSION_NUM,\n",
     "    _SHEET_TYPE_KEY: sheet_type,\n",
+    "    _SHEET_VERSION_KEY: sheet_version,\n",
     "    _EXPERIMENT_NAME_KEY: curr_run_title,\n",
     "    _BIOINFORMATICS_KEY:  [],\n",
     "    _CONTACT_KEY: [],\n",
@@ -484,7 +490,7 @@
     "    metadata[_CONTACT_KEY].append(\n",
     "        {\n",
     "            'Sample_Project': unique_projects[0],\n",
-    "            'Email': \"gail.ackermann50@gmail.com\"\n",
+    "            'Email': contact_email\n",
     "        }\n",
     "    )\n",
     "# next qiita id\n",