Utf8 regression (#1000)

* BUG: unable to parse non-trivial UTF-8 characters

* BUG: use .asstr per h5py recommendations ('https://docs.h5py.org/en/latest/strings.html#reading-strings')

* asstr does not handle an empty dataset

* MAINT: in1d was deprecated in numpy switch to isin

* DOC: changelog mention

* macos-13 no longer supported

Author: wasade

.github/workflows/python-package-conda.yml

@@ -81,7 +81,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest, ubuntu-24.04-arm, macos-13]
+        os: [ubuntu-latest, macos-latest, windows-latest, ubuntu-24.04-arm, macos-15-intel]
         python_version: ${{ fromJSON(needs.conf.outputs.supported_pythons) }}
         use_conda: [true, false]
     steps:

ChangeLog.md

@@ -1,6 +1,13 @@
 BIOM-Format ChangeLog
 =====================
 
+biom-2.1.18-dev
+---------------
+
+Bug fixes:
+
+* A regression with parse of UTF-8 characters was introduced, reported [here](https://forum.qiime2.org/t/feature-table-gives-ascii-codec-cant-decode-byte-0xc3-in-position-4-error/33858/7)
+
 biom-2.1.17
 -----------
 

biom/table.py

@@ -4182,11 +4182,15 @@ def ensure_utf8(x):
         def axis_load(grp):
             """Loads all the data of the given group"""
             # fetch all of the IDs
-            ids = grp['ids'][:]
+            ids = grp['ids']
 
             if ids.size > 0:
+                ids = ids.asstr()[:]
                 ids_dtype = 'U%d' % max([len(v) for v in ids])
                 ids = np.asarray(ids, dtype=ids_dtype)
+            else:
+                # .asstr does not handle an empty dataset
+                ids = ids[:]
 
             parser = defaultdict(lambda: general_parser)
             parser['taxonomy'] = vlen_list_of_str_parser
@@ -4234,7 +4238,7 @@ def _get_ids(source_ids, desired_ids):
                 else:
                     desired_ids = np.asarray(desired_ids)
                     # Get the index of the source ids to include
-                    idx = np.in1d(source_ids, desired_ids)
+                    idx = np.isin(source_ids, desired_ids)
                     # Retrieve only the ids that we are interested on
                     ids = source_ids[idx]
                     # Check that all desired ids have been found on source ids

biom/tests/test_parse.py

@@ -17,6 +17,7 @@
 import numpy as np
 import numpy.testing as npt
 
+from biom import example_table
 from biom.parse import (generatedby, MetadataMap, parse_biom_table, parse_uc,
                         load_table, save_table)
 from biom.table import Table
@@ -283,6 +284,15 @@ def test_parse_adjacency_table_no_header(self):
         obs = Table.from_adjacency(''.join(lines))
         self.assertEqual(obs, exp)
 
+    def test_parse_biom_table_hdf5_accent_utf8_regression(self):
+        tab = example_table.copy()
+        tab.update_ids({i: f'{i}fóó' for i in tab.ids()}, inplace=True)
+        with NamedTemporaryFile(delete=False) as tmpfile:
+            with h5py.File(tmpfile.name, 'w') as fp:
+                tab.to_hdf5(fp, 'asf')
+            obs = load_table(tmpfile.name)
+            self.assertEqual(obs, tab)
+
     def test_parse_biom_table_hdf5(self):
         """Make sure we can parse a HDF5 table through the same loader"""
         cwd = os.getcwd()