MAINT: move Importance formats and type to q2-types

q2_types/feature_data/__init__.py

@@ -28,12 +28,14 @@
     MixedCaseAlignedRNAFASTAFormat,
     MixedCaseAlignedRNASequencesDirectoryFormat,
     SequenceCharacteristicsDirectoryFormat,
-    SequenceCharacteristicsFormat)
+    SequenceCharacteristicsFormat, ImportanceFormat,
+    ImportanceDirectoryFormat)
 from ._types import (
     FeatureData, Taxonomy, Sequence, LinkedSequence, PairedEndSequence,
     AlignedSequence,
     Differential, ProteinSequence, AlignedProteinSequence, RNASequence,
-    AlignedRNASequence, PairedEndRNASequence, BLAST6, SequenceCharacteristics)
+    AlignedRNASequence, PairedEndRNASequence, BLAST6,
+    SequenceCharacteristics, Importance)
 from ._objects import (
     NucleicAcidIterator, DNAIterator, PairedDNAIterator, AlignedDNAIterator,
     ProteinIterator, AlignedProteinIterator, RNAIterator, AlignedRNAIterator,
@@ -72,5 +74,6 @@
     'MixedCaseProteinSequencesDirectoryFormat',
     'MixedCaseAlignedProteinSequencesDirectoryFormat',
     'SequenceCharacteristics', 'SequenceCharacteristicsDirectoryFormat',
-    'SequenceCharacteristicsFormat',
+    'SequenceCharacteristicsFormat', 'Importance', 'ImportanceFormat',
+    'ImportanceDirectoryFormat',
 ]

q2_types/feature_data/_deferred_setup/__init__.py

@@ -31,10 +31,12 @@
     MixedCaseAlignedRNASequencesDirectoryFormat,
     SequenceCharacteristicsDirectoryFormat,
     SequenceCharacteristicsFormat,
+    ImportanceFormat, ImportanceDirectoryFormat,
     FeatureData, Taxonomy, Sequence, LinkedSequence, PairedEndSequence,
     AlignedSequence,
     Differential, ProteinSequence, AlignedProteinSequence, RNASequence,
-    AlignedRNASequence, PairedEndRNASequence, BLAST6, SequenceCharacteristics)
+    AlignedRNASequence, PairedEndRNASequence, BLAST6,
+    SequenceCharacteristics, Importance)
 
 from ...plugin_setup import plugin
 
@@ -60,7 +62,8 @@
     MixedCaseAlignedDNASequencesDirectoryFormat,
     MixedCaseAlignedRNAFASTAFormat,
     MixedCaseAlignedRNASequencesDirectoryFormat, SequenceCharacteristicsFormat,
-    SequenceCharacteristicsDirectoryFormat
+    SequenceCharacteristicsDirectoryFormat, ImportanceFormat,
+    ImportanceDirectoryFormat
 )
 
 plugin.register_semantic_types(FeatureData, Taxonomy, Sequence,
@@ -69,7 +72,7 @@
                                Differential, ProteinSequence,
                                AlignedProteinSequence, RNASequence,
                                AlignedRNASequence, PairedEndRNASequence,
-                               BLAST6, SequenceCharacteristics)
+                               BLAST6, SequenceCharacteristics, Importance)
 
 plugin.register_artifact_class(
     FeatureData[Taxonomy],
@@ -165,5 +168,11 @@
     description=("Characteristics of sequences (e.g., the length of a gene "
                  "in basepairs)."))
 
+plugin.register_artifact_class(
+    FeatureData[Importance],
+    directory_format=ImportanceDirectoryFormat,
+    description=("Numeric importance scores associated with a set of feature "
+                 "identifiers."))
+
 importlib.import_module('._transformers', __name__)
 importlib.import_module('._validators', __name__)

q2_types/feature_data/_deferred_setup/_transformers.py

@@ -11,6 +11,7 @@
 import pandas as pd
 import biom
 import skbio
+import numpy as np
 
 import qiime2
 
@@ -25,7 +26,7 @@
     AlignedRNAFASTAFormat, PairedRNASequencesDirectoryFormat,
     BLAST6Format, MixedCaseDNAFASTAFormat, MixedCaseRNAFASTAFormat,
     MixedCaseAlignedDNAFASTAFormat, MixedCaseAlignedRNAFASTAFormat,
-    SequenceCharacteristicsFormat,
+    SequenceCharacteristicsFormat, ImportanceFormat,
     DNAIterator, PairedDNAIterator, AlignedDNAIterator,
     ProteinIterator, AlignedProteinIterator, RNAIterator, AlignedRNAIterator,
     PairedRNAIterator
@@ -110,6 +111,14 @@ def _has_expected_header(df):
     return df.iloc[0].tolist()[:2] == TSVTaxonomyFormat.HEADER
 
 
+def _read_dataframe(fh):
+    """Read a TSV file into a dataframe with the first column as the index."""
+    df = pd.read_csv(fh, sep='\t', header=0, dtype='str')
+    df.set_index(df.columns[0], drop=True, append=False, inplace=True)
+    df.index.name = 'id'
+    return df
+
+
 def _dataframe_to_tsv_taxonomy_format(df):
     if len(df.index) < 1:
         raise ValueError("Taxonomy format requires at least one row of data.")
@@ -242,6 +251,31 @@ def _29(ff: TSVTaxonomyFormat) -> qiime2.Metadata:
     return qiime2.Metadata(df)
 
 
+@plugin.register_transformer
+def _dataframe_to_importance_format(data: pd.DataFrame) -> ImportanceFormat:
+    """Write feature importance values from a dataframe to TSV format."""
+    ff = ImportanceFormat()
+    with ff.open() as fh:
+        data.to_csv(fh, sep='\t', header=True, na_rep=np.nan)
+    return ff
+
+
+@plugin.register_transformer
+def _importance_format_to_dataframe(ff: ImportanceFormat) -> pd.DataFrame:
+    """Read feature importance TSV values into a numeric dataframe."""
+    with ff.open() as fh:
+        return _read_dataframe(fh).apply(
+            lambda x: pd.to_numeric(x, errors='raise'))
+
+
+@plugin.register_transformer
+def _importance_format_to_metadata(ff: ImportanceFormat) -> qiime2.Metadata:
+    """Read feature importance TSV values into QIIME 2 metadata."""
+    with ff.open() as fh:
+        return qiime2.Metadata(_read_dataframe(fh).apply(
+            lambda x: pd.to_numeric(x, errors='raise')))
+
+
 @plugin.register_transformer
 def _24(df: pd.DataFrame) -> TSVTaxonomyFormat:
     return _dataframe_to_tsv_taxonomy_format(df)

q2_types/feature_data/_formats.py

@@ -522,3 +522,50 @@ def validate(self, n_records=None):
     "SequenceCharacteristicsDirectoryFormat",
     "sequence_characteristics.tsv", SequenceCharacteristicsFormat
 )
+
+
+class ImportanceFormat(model.TextFileFormat):
+    def _validate(self, n_records=None):
+        """Validate rows with an identifier followed by numeric values."""
+        with self.open() as fh:
+            # The header is intentionally flexible because column names and
+            # counts can vary by estimator.
+            fh.readline()
+
+            has_data = False
+            for line_number, line in enumerate(fh, start=2):
+                # Strip each cell rather than the full line so empty cells are
+                # preserved and fail numeric validation.
+                cells = [c.strip() for c in line.split('\t')]
+                if len(cells) < 2:
+                    raise ValidationError(
+                        "Expected data record to be TSV with two or more "
+                        "fields. Detected {0} fields at line {1}:\n\n{2!r}"
+                        .format(len(cells), line_number, cells))
+
+                try:
+                    [float(c) for c in cells[1:]]
+                except ValueError:
+                    raise ValidationError(
+                        "Columns must contain only numeric values. "
+                        "A non-numeric value ({0!r}) was detected at line "
+                        "{1}.".format(cells[1], line_number))
+
+                has_data = True
+                if n_records is not None and (line_number - 1) >= n_records:
+                    break
+
+            if not has_data:
+                raise ValidationError(
+                    "There must be at least one data record present in the "
+                    "file in addition to the header line.")
+
+    def _validate_(self, level):
+        """Validate this format using QIIME 2's min or max validation level."""
+        record_count_map = {'min': 5, 'max': None}
+        self._validate(record_count_map[level])
+
+
+ImportanceDirectoryFormat = model.SingleFileDirectoryFormat(
+    'ImportanceDirectoryFormat', 'importance.tsv',
+    ImportanceFormat)

q2_types/feature_data/_types.py

@@ -45,3 +45,5 @@
 
 SequenceCharacteristics = SemanticType('SequenceCharacteristics',
                                        variant_of=FeatureData.field['type'])
+
+Importance = SemanticType('Importance', variant_of=FeatureData.field['type'])

q2_types/feature_data/tests/data/chardonnay.map.txt

@@ -0,0 +1,22 @@
+#SampleID	Vineyard	Region	latitude	longitude
+v4.3.618	1	Coombsville	38.306	-122.228
+v4.3.642	1	Coombsville	38.306	-122.228
+v4.3.687	1	Coombsville	38.306	-122.228
+v4.3.412	2	Coombsville	38.306	-122.227
+v4.3.637	2	Coombsville	38.306	-122.227
+v4.3.643	2	Coombsville	38.306	-122.227
+v4.3.350	3	Carneros	38.212	-122.213
+v4.3.353	3	Carneros	38.212	-122.213
+v4.3.355	3	Carneros	38.212	-122.213
+v4.3.415	8	Oakville	38.418	-122.412
+v4.3.416	8	Oakville	38.418	-122.412
+v4.3.417	8	Oakville	38.418	-122.412
+v4.3.692	9	Carneros	38.285	-122.322
+v4.3.693	9	Carneros	38.285	-122.322
+v4.3.694	9	Carneros	38.285	-122.322
+v4.3.600	10	Russian_River	38.424	-122.895
+v4.3.657	10	Russian_River	38.424	-122.895
+v4.3.659	10	Russian_River	38.424	-122.895
+v4.3.587	11	Russian_River	38.455	-122.862
+v4.3.588	11	Russian_River	38.455	-122.862
+v4.3.589	11	Russian_River	38.455	-122.862

q2_types/feature_data/tests/data/empty_file.txt

@@ -0,0 +1 @@
+#SampleID	Garbage

q2_types/feature_data/tests/data/garbage.txt

@@ -0,0 +1,6 @@
+#SampleID
+one
+two
+buckle
+my
+shoe