Merge pull request #1514 from Unidata/post1.7.5_testfix

fix test_cli, turn on zstd compression filter test.

Author: jswhit

.github/workflows/build_old.yml

@@ -12,6 +12,7 @@ jobs:
       #CC: mpicc.mpich
       CC: mpicc
       #NO_NET: 1
+      SKIP_FILTER_TEST: 1
     strategy:
       matrix:
         python-version: ["3.14"]

Changelog

@@ -1,3 +1,10 @@
+ version 1.7.5.1 (tag v1.7.5.1rel)
+ =================================
+ * make sure zstd compression filter test always runs unless specifically disabled, fix cli test so it runs on
+ 32-bit x86.  New wheels for linux that include plugins (they were left out of the 1.7.5 release).
+ * fix version comparison logic that was broken when using netcdf-c >= 4.10.0 - this was the cause of 
+ issue #1515 (diskless persisted datasets are no longer persisted).
+
  version 1.7.5 (tag v1.7.5rel)
  =============================
  * remove python 3.10 support (it reaches EOL October 31, 2026).

pyproject.toml

@@ -182,7 +182,8 @@ environment = {HDF5_DIR='C:\\\\Users\\runneradmin\\micromamba\\envs\\build\\Libr
 [[tool.cibuildwheel.overrides]]
 select = "*-win_arm64"
 inherit.environment = "append"
-environment = { HDF5_DIR = 'C:\\\\vcpkg\\\\installed\\\\arm64-windows', netCDF4_DIR = 'C:\\\\vcpkg\\\\installed\\\\arm64-windows', PATH = 'C:\\\\vcpkg\\\\installed\\\\arm64-windows\\\\bin;${PATH}', NO_CDL = '1' }
+# no filter plugins on windows arm64
+environment = { SKIP_FILTER_TEST=1, HDF5_DIR = 'C:\\\\vcpkg\\\\installed\\\\arm64-windows', netCDF4_DIR = 'C:\\\\vcpkg\\\\installed\\\\arm64-windows', PATH = 'C:\\\\vcpkg\\\\installed\\\\arm64-windows\\\\bin;${PATH}', NO_CDL = '1' }
 repair-wheel-command = [
   "delvewheel show {wheel}",
   "delvewheel repair -w {dest_dir} {wheel}",

src/netCDF4/_netCDF4.pyx

@@ -1,4 +1,4 @@
-"""Version 1.7.5
+"""Version 1.7.5.1
 -------------
 
 # Introduction
@@ -1283,7 +1283,7 @@ import sys
 import functools
 from typing import Union
 
-__version__ = "1.7.5"
+__version__ = "1.7.5.1"
 
 # Initialize numpy
 import posixpath
@@ -1493,7 +1493,9 @@ def set_alignment(threshold, alignment):
 
 __netcdf4libversion__ = getlibversion().split()[0]
 __hdf5libversion__ = _gethdf5libversion()
-_needsworkaround_issue485 = __netcdf4libversion__ < "4.4.0" or \
+# parsed lib version for comparisons (string comparison breaks for 4.10+)
+_nclibversion = Version(__netcdf4libversion__.split('-')[0])
+_needsworkaround_issue485 = _nclibversion < Version("4.4.0") or \
                (__netcdf4libversion__.startswith("4.4.0") and \
                 "-development" in __netcdf4libversion__)
 
@@ -1605,7 +1607,7 @@ is_native_big = numpy.dtype('>f4').byteorder == c'='
 # so it will compile with versions <= 4.2.
 NC_DISKLESS = 0x0008
 # introduced in 4.6.2
-if __netcdf4libversion__[0:5] >= "4.6.2":
+if _nclibversion >= Version("4.6.2"):
     NC_PERSIST = 0x4000
 else:  # prior to 4.6.2 this flag doesn't work, so make the same as NC_DISKLESS
     NC_PERSIST = NC_DISKLESS
@@ -4499,7 +4501,7 @@ does not do anything."""
         # a fix for strided slice access using HDF5 was added
         # in 4.6.2.
         # always use nc_get_vars for strided access with OpenDAP (issue #838).
-        if __netcdf4libversion__ >= "4.6.2" or\
+        if _nclibversion >= Version("4.6.2") or\
            self._grp.filepath().startswith('http'):
             self._use_get_vars = True
         else:
@@ -4574,7 +4576,7 @@ does not do anything."""
         if (self._grp.path != '/'): ncdump.append('path = %s' % self._grp.path)
         ncdump.append('unlimited dimensions: %s' % ', '.join(unlimdims))
         ncdump.append('current shape = %r' % (self.shape,))
-        if __netcdf4libversion__ < '4.5.1' and\
+        if _nclibversion < Version('4.5.1') and\
             self._grp.file_format.startswith('NETCDF3'):
             # issue #908: no_fill not correct for NETCDF3 files before 4.5.1
             # before 4.5.1 there was no way to turn off filling on a
@@ -5267,7 +5269,7 @@ rename a `Variable` attribute named `oldname` to `newname`."""
         # issue 209: don't return masked array if variable filling
         # is disabled.
         else:
-            if __netcdf4libversion__ < '4.5.1' and\
+            if _nclibversion < Version('4.5.1') and\
                 self._grp.file_format.startswith('NETCDF3'):
                 # issue #908: no_fill not correct for NETCDF3 files before 4.5.1
                 # before 4.5.1 there was no way to turn off filling on a

test/parse_clean_version.py

@@ -0,0 +1,16 @@
+from packaging.version import Version, InvalidVersion
+import re
+
+def parse_clean_version(version_str: str) -> Version:
+    # Normalize "-development" or any variation like "-dev" to ".dev0"
+    cleaned = re.sub(r'[-_.]?development', '.dev0', version_str, flags=re.IGNORECASE)
+    # Catch cases where it's just raw text hyphens (e.g., 1.0.0-dev -> 1.0.0.dev0)
+    cleaned = re.sub(r'-dev(\d*)', r'.dev\1', cleaned, flags=re.IGNORECASE)
+    try:
+        return Version(cleaned)
+    except InvalidVersion:
+        # Fallback regex to pull just the numerical digits if it's still breaking
+        match = re.search(r'(\d+(\.\d+)+)', version_str)
+        if match:
+            return Version(match.group(1))
+        raise

test/test_cli.py

@@ -77,5 +77,8 @@ def test_nc4tonc3_chunk(tmp_path, chunk, monkeypatch):
         utils.nc4tonc3()
     assert convert.call_args.kwargs["nchunk"] == (10 if chunk is None else chunk)
     with Dataset(destination) as dataset:
-        assert dataset.file_format == "NETCDF3_64BIT_OFFSET"
+        if sys.maxsize > 2**32:
+           assert dataset.file_format == "NETCDF3_64BIT_OFFSET"
+        else:
+           assert dataset.file_format == "NETCDF3_64BIT"
         np.testing.assert_array_equal(dataset.variables["values"][:], values)

test/test_compression_zstd.py

@@ -23,7 +23,9 @@ def write_netcdf(filename,dtype='f8',complevel: CompressionLevel = 6):
     nc.close()
 
 
-@unittest.skipIf(no_plugins or not has_zstd_filter, "zstd filter not available")
+# always run zstd test unless SKIP_FILTER_TEST set, so we know if filters are installed in wheels.
+@unittest.skipIf(os.getenv("SKIP_FILTER_TEST"), "skipping zstd filter test")
+#@unittest.skipIf(no_plugins or not has_zstd_filter, "zstd filter not available")
 class CompressionTestCase(unittest.TestCase):
     def setUp(self):
         self.filename1 = filename1

test/test_diskless.py

@@ -1,4 +1,6 @@
 import unittest, os, tempfile
+from packaging.version import Version
+from parse_clean_version import parse_clean_version
 import numpy as np
 from numpy.random.mtrand import uniform
 from numpy.testing import assert_array_equal, assert_array_almost_equal
@@ -17,7 +19,7 @@
 
 
 @unittest.skipIf(
-    netCDF4.__netcdf4libversion__ < "4.2.1"
+    parse_clean_version(netCDF4.__netcdf4libversion__) < Version('4.2.1')
     or netCDF4.__has_parallel4_support__
     or netCDF4.__has_pnetcdf_support__,
     "no diskless support",
@@ -74,7 +76,7 @@ def runTest(self):
         assert os.path.isfile(self.file)==False 
         # open persisted file.
         # first, check that file does actually exist on disk
-        assert os.path.isfile(self.file2)==True 
+        assert os.path.isfile(self.file2) and os.path.getsize(self.file2) > 0
         f = netCDF4.Dataset(self.file2)
         foo = f.variables['data1']
         # check shape.

test/test_masked5.py

@@ -1,6 +1,8 @@
 import unittest
 import os
 import tempfile
+from packaging.version import Version
+from parse_clean_version import parse_clean_version
 
 import numpy as np
 from numpy import ma
@@ -55,7 +57,7 @@ def test_scaled(self):
         # issue 730
         # this part fails with netcdf 4.1.3
         # a bug in vlen strings?
-        if __netcdf4libversion__ >= '4.4.0':
+        if parse_clean_version(__netcdf4libversion__) >= Version('4.4.0'):
             assert v2[0] == 'first'
             assert v2[1] == '<missing>'
 

test/test_vlen.py

@@ -2,9 +2,11 @@
 import unittest
 import os
 import tempfile
+from packaging.version import Version
 from netCDF4 import Dataset
 import numpy as np
 from numpy.testing import assert_array_equal
+from parse_clean_version import parse_clean_version
 
 FILE_NAME = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name
 VL_NAME = 'vlen_type'
@@ -163,7 +165,7 @@ class VlenAppendTestCase(unittest.TestCase):
     def setUp(self):
 
         import netCDF4
-        if netCDF4.__netcdf4libversion__ < "4.4.1":
+        if parse_clean_version(netCDF4.__netcdf4libversion__) < Version("4.4.1"):
             self.skip = True
             try:
                 self.skipTest("This test requires NetCDF 4.4.1 or later.")