Skip to content

Commit dd70fcb

Browse files
jswhit2claude
authored andcommitted
add extra_mode_flag Dataset kwarg for user-defined formats (issue #1490)
Add an extra_mode_flag kwarg to Dataset that is ORed into the mode passed to nc_open, so files without a magic number can be opened with a user-defined format (UDF) dispatch layer, such as those provided by the NetCDF Expansion Pack (NEP). Add NEP_UDF_* module constants (names match nep.h). Only allowed for plain read-only opens (mode='r', not diskless, memory or parallel), requires netcdf-c >= 4.10.0. Add tests for the kwarg (opening an mmCIF file with no magic number) and for reading GRIB2 with NEP (test/gfs_sample.grb2, the first 3 messages of pygrib's sampledata/gfs.grb). The NEP read tests are skipped unless the NEP readers are registered with netcdf-c. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 20652b4 commit dd70fcb

7 files changed

Lines changed: 243 additions & 2 deletions

File tree

‎MANIFEST.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ include examples/README.md
1010
exclude examples/data
1111
include test/*py
1212
include test/*nc
13+
include test/*grb2
1314
include src/netCDF4/__init__.py
1415
include src/netCDF4/_netCDF4.pyx
1516
exclude src/netCDF4/_netCDF4.c

‎src/netCDF4/__init__.pyi‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,16 @@ is_native_big: bool
187187
default_encoding: Final = "utf-8"
188188
unicode_error: Final = "replace"
189189
default_fillvals: dict[str, int | float | str]
190+
NEP_UDF_GEOTIFF_BIGTIFF: Final = 0x0040
191+
NEP_UDF_GEOTIFF_STANDARD: Final = 0x0080
192+
NEP_UDF_GRIB2: Final = 0x10000
193+
NEP_UDF_FITS: Final = 0x80000
194+
NEP_UDF_CDF: Final = 0x100000
195+
NEP_UDF_PDS4: Final = 0x200000
196+
NEP_UDF_DICOM: Final = 0x400000
197+
NEP_UDF_PDB: Final = 0x800000
198+
NEP_UDF_MMCIF: Final = 0x1000000
199+
NEP_UDF_NEXTCDF4: Final = 0x2000000
190200

191201
# date2index, date2num, and num2date are actually provided by cftime and if stubs for
192202
# cftime are completed these should be removed.
@@ -235,6 +245,7 @@ class Dataset:
235245
comm: Any = None,
236246
info: Any = None,
237247
auto_complex: bool = False,
248+
extra_mode_flag: int | None = None,
238249
**kwargs: Any,
239250
): ...
240251
@property

‎src/netCDF4/_netCDF4.pyi‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,16 @@ from . import (
88
Group,
99
MFDataset,
1010
MFTime,
11+
NEP_UDF_CDF,
12+
NEP_UDF_DICOM,
13+
NEP_UDF_FITS,
14+
NEP_UDF_GEOTIFF_BIGTIFF,
15+
NEP_UDF_GEOTIFF_STANDARD,
16+
NEP_UDF_GRIB2,
17+
NEP_UDF_MMCIF,
18+
NEP_UDF_NEXTCDF4,
19+
NEP_UDF_PDB,
20+
NEP_UDF_PDS4,
1121
NetCDF4MissingFeatureException,
1222
Variable,
1323
VLType,

‎src/netCDF4/_netCDF4.pyx‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1612,6 +1612,23 @@ if _nclibversion >= Version("4.6.2"):
16121612
else: # prior to 4.6.2 this flag doesn't work, so make the same as NC_DISKLESS
16131613
NC_PERSIST = NC_DISKLESS
16141614

1615+
# mode flags for the user-defined formats (UDFs) provided by the
1616+
# NetCDF Expansion Pack (NEP), names match those in nep.h.
1617+
# Used with the extra_mode_flag Dataset kwarg (requires netcdf-c >= 4.10.0).
1618+
NEP_UDF_GEOTIFF_BIGTIFF = 0x0040 # NC_UDF0
1619+
NEP_UDF_GEOTIFF_STANDARD = 0x0080 # NC_UDF1
1620+
NEP_UDF_GRIB2 = 0x10000 # NC_UDF2
1621+
NEP_UDF_FITS = 0x80000 # NC_UDF3
1622+
NEP_UDF_CDF = 0x100000 # NC_UDF4
1623+
NEP_UDF_PDS4 = 0x200000 # NC_UDF5
1624+
NEP_UDF_DICOM = 0x400000 # NC_UDF6
1625+
NEP_UDF_PDB = 0x800000 # NC_UDF7
1626+
NEP_UDF_MMCIF = 0x1000000 # NC_UDF8
1627+
NEP_UDF_NEXTCDF4 = 0x2000000 # NC_UDF9
1628+
_nep_udf_flags = (NEP_UDF_GEOTIFF_BIGTIFF, NEP_UDF_GEOTIFF_STANDARD,
1629+
NEP_UDF_GRIB2, NEP_UDF_FITS, NEP_UDF_CDF, NEP_UDF_PDS4,
1630+
NEP_UDF_DICOM, NEP_UDF_PDB, NEP_UDF_MMCIF, NEP_UDF_NEXTCDF4)
1631+
16151632
# next two lines do nothing, preserved for backwards compatibility.
16161633
default_encoding = 'utf-8'
16171634
unicode_error = 'replace'
@@ -2306,11 +2323,13 @@ strings.
23062323
def __init__(self, filename, mode='r', clobber=True, format='NETCDF4',
23072324
diskless=False, persist=False, keepweakref=False,
23082325
memory=None, encoding=None, parallel=False,
2309-
comm=None, info=None, auto_complex=False, **kwargs):
2326+
comm=None, info=None, auto_complex=False,
2327+
extra_mode_flag=None, **kwargs):
23102328
"""
23112329
**`__init__(self, filename, mode="r", clobber=True, diskless=False,
23122330
persist=False, keepweakref=False, memory=None, encoding=None,
2313-
parallel=False, comm=None, info=None, format='NETCDF4')`**
2331+
parallel=False, comm=None, info=None, format='NETCDF4',
2332+
auto_complex=False, extra_mode_flag=None)`**
23142333
23152334
`Dataset` constructor.
23162335
@@ -2403,6 +2422,15 @@ strings.
24032422
means MPI_INFO_NULL will be used. Ignored if `parallel=False`.
24042423
24052424
**`auto_complex`**: if `True`, then automatically convert complex number types
2425+
2426+
**`extra_mode_flag`**: integer mode flag selecting a user-defined format
2427+
(UDF) dispatch layer, such as those provided by the NetCDF Expansion Pack
2428+
(NEP). Only needed for files that cannot be identified by their magic
2429+
number. Must be one of the `NEP_UDF_*` module constants (for example
2430+
`NEP_UDF_MMCIF`), and the UDF must be registered with netcdf-c (for NEP,
2431+
via the .ncrc file). Only allowed with `mode='r'`, and not with
2432+
`diskless`, `memory` or `parallel`. Requires netcdf-c >= 4.10.0.
2433+
Default `None`.
24062434
"""
24072435
cdef int grpid, ierr, numgrps, numdims, numvars,
24082436
cdef size_t initialsize
@@ -2478,6 +2506,19 @@ strings.
24782506
if write_mode and persist:
24792507
cmode |= NC_WRITE | NC_PERSIST
24802508

2509+
# extra mode flag for user-defined formats, only used for plain
2510+
# read-only nc_open (cmode == NC_NOWRITE). Need to check read_mode
2511+
# too, since NC_CLOBBER (mode='w') is also zero.
2512+
if extra_mode_flag is not None:
2513+
if _nclibversion < Version("4.10.0"):
2514+
raise NetCDF4MissingFeatureException("extra_mode_flag", "4.10.0")
2515+
if not read_mode or cmode != NC_NOWRITE or memory is not None or parallel:
2516+
raise ValueError("extra_mode_flag only allowed with mode='r', "
2517+
"and not with diskless, memory or parallel")
2518+
if extra_mode_flag not in _nep_udf_flags:
2519+
raise ValueError(f"extra_mode_flag must be one of the NEP_UDF_* "
2520+
f"constants, got {extra_mode_flag!r}")
2521+
24812522
if write_mode:
24822523
_set_default_format(format=format)
24832524
if memory is not None:
@@ -2521,6 +2562,8 @@ strings.
25212562
with nogil:
25222563
ierr = nc_open_par(path, cmode | NC_MPIIO, mpicomm, mpiinfo, &grpid)
25232564
else:
2565+
if extra_mode_flag is not None:
2566+
cmode |= extra_mode_flag
25242567
with nogil:
25252568
ierr = nc_open(path, cmode, &grpid)
25262569

‎test/gfs_sample.grb2‎

26.7 KB
Binary file not shown.

‎test/test_extra_mode_flag.py‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
import unittest, os, tempfile
2+
from packaging.version import Version
3+
from numpy.testing import assert_array_almost_equal
4+
import netCDF4
5+
6+
# test extra_mode_flag Dataset kwarg (issue #1490), used to select a
7+
# user-defined format (UDF) dispatch layer for files with no magic number.
8+
# The read tests only run if the NetCDF Expansion Pack (NEP) mmCIF reader
9+
# is registered with netcdf-c (e.g. via NCRCENV_RC pointing to NEP's .ncrc).
10+
11+
MMCIF = """\
12+
data_TINY
13+
#
14+
_entry.id TINY
15+
#
16+
loop_
17+
_atom_site.group_PDB
18+
_atom_site.id
19+
_atom_site.type_symbol
20+
_atom_site.label_atom_id
21+
_atom_site.label_alt_id
22+
_atom_site.label_comp_id
23+
_atom_site.label_asym_id
24+
_atom_site.label_entity_id
25+
_atom_site.label_seq_id
26+
_atom_site.pdbx_PDB_ins_code
27+
_atom_site.Cartn_x
28+
_atom_site.Cartn_y
29+
_atom_site.Cartn_z
30+
_atom_site.occupancy
31+
_atom_site.B_iso_or_equiv
32+
_atom_site.auth_seq_id
33+
_atom_site.auth_asym_id
34+
_atom_site.pdbx_PDB_model_num
35+
ATOM 1 N N . VAL A 1 1 ? 6.204 16.869 4.854 1.00 49.05 1 A 1
36+
ATOM 2 C CA . VAL A 1 1 ? 6.913 17.759 4.607 1.00 43.14 1 A 1
37+
ATOM 3 C C . VAL A 1 1 ? 8.504 17.378 4.797 1.00 24.80 1 A 1
38+
#
39+
"""
40+
CARTN_X = [6.204, 6.913, 8.504]
41+
42+
has_udf = Version(netCDF4.__netcdf4libversion__.split('-')[0]) >= Version('4.10.0')
43+
44+
def _write(text):
45+
f = tempfile.NamedTemporaryFile(suffix='.cif', delete=False, mode='w')
46+
f.write(text)
47+
f.close()
48+
return f.name
49+
50+
def _has_nep_mmcif():
51+
if not has_udf:
52+
return False
53+
filename = _write(MMCIF)
54+
try:
55+
netCDF4.Dataset(filename).close()
56+
return True
57+
except OSError:
58+
return False
59+
finally:
60+
os.remove(filename)
61+
62+
has_nep_mmcif = _has_nep_mmcif()
63+
64+
65+
@unittest.skipUnless(has_udf, "extra_mode_flag requires netcdf-c >= 4.10.0")
66+
class ExtraModeFlagArgsTestCase(unittest.TestCase):
67+
68+
def setUp(self):
69+
self.file = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name
70+
netCDF4.Dataset(self.file, 'w').close()
71+
72+
def tearDown(self):
73+
os.remove(self.file)
74+
75+
def runTest(self):
76+
"""only allowed for plain read-only opens with a NEP_UDF_* flag"""
77+
flag = netCDF4.NEP_UDF_MMCIF
78+
for kwargs in (dict(mode='w'), dict(mode='a'), dict(mode='r+'),
79+
dict(mode='rs'), dict(mode='r', diskless=True)):
80+
with self.assertRaises(ValueError, msg=str(kwargs)):
81+
netCDF4.Dataset(self.file, extra_mode_flag=flag, **kwargs)
82+
with open(self.file, 'rb') as f:
83+
buf = f.read()
84+
with self.assertRaises(ValueError):
85+
netCDF4.Dataset('inmemory.nc', memory=buf, extra_mode_flag=flag)
86+
# not a NEP_UDF_* flag (NC_WRITE)
87+
with self.assertRaises(ValueError):
88+
netCDF4.Dataset(self.file, extra_mode_flag=0x0001)
89+
# extra_mode_flag=None is the default
90+
netCDF4.Dataset(self.file, extra_mode_flag=None).close()
91+
92+
93+
@unittest.skipUnless(has_nep_mmcif, "NEP mmCIF reader not available")
94+
class ExtraModeFlagNEPTestCase(unittest.TestCase):
95+
96+
def setUp(self):
97+
# without the leading data_ magic number netcdf-c can't detect the format
98+
self.file = _write("# no magic number\n" + MMCIF)
99+
100+
def tearDown(self):
101+
os.remove(self.file)
102+
103+
def runTest(self):
104+
"""open mmCIF file with no magic number using NEP_UDF_MMCIF"""
105+
with self.assertRaises(OSError):
106+
netCDF4.Dataset(self.file)
107+
with netCDF4.Dataset(self.file, extra_mode_flag=netCDF4.NEP_UDF_MMCIF) as ds:
108+
assert ds.entry_id == 'TINY'
109+
assert len(ds.dimensions['atom']) == 3
110+
assert_array_almost_equal(ds['atom_site_Cartn_x'][0], CARTN_X)
111+
112+
113+
if __name__ == '__main__':
114+
unittest.main()

‎test/test_nep_grib2.py‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
import unittest, os
2+
import numpy as np
3+
from numpy.testing import assert_allclose
4+
import netCDF4
5+
6+
# test reading a GRIB2 file with the NetCDF Expansion Pack (NEP) GRIB2
7+
# reader (issue #1490). Only runs if the NEP GRIB2 reader is registered
8+
# with netcdf-c (e.g. via NCRCENV_RC pointing to NEP's .ncrc).
9+
# gfs_sample.grb2 is the first 3 messages of pygrib's sampledata/gfs.grb
10+
# (10 hPa HGT, TMP and RH on a 2.5 degree global grid).
11+
12+
FILE_NAME = os.path.join(os.path.dirname(__file__), 'gfs_sample.grb2')
13+
NLATS, NLONS = 73, 144
14+
15+
# reference values from pygrib: (value at [0,0], [36,72], [-1,-1], min, max)
16+
REFERENCE = {
17+
'HGT': (29989.89, 31001.2, 27951.64, 27900.99, 31664.09),
18+
'TMP': (213.7, 233.1, 221.0, 209.3, 257.1),
19+
'RH': (0.0, 0.03, 0.06, 0.0, 0.21),
20+
}
21+
22+
def _has_nep_grib2():
23+
try:
24+
netCDF4.Dataset(FILE_NAME).close()
25+
return True
26+
except OSError:
27+
return False
28+
29+
has_nep_grib2 = _has_nep_grib2()
30+
31+
32+
@unittest.skipUnless(has_nep_grib2, "NEP GRIB2 reader not available")
33+
class NEPGrib2TestCase(unittest.TestCase):
34+
35+
def check(self, ds):
36+
assert ds.Conventions == 'GRIB2'
37+
assert ds.GRIB2_edition == 2
38+
assert list(ds.variables) == list(REFERENCE)
39+
for name, (v00, vmid, vlast, vmin, vmax) in REFERENCE.items():
40+
var = ds[name]
41+
assert var.long_name == name
42+
data = var[:]
43+
assert data.size == NLATS * NLONS
44+
# NEP currently defines the dimensions in the wrong order
45+
# (y=144, x=73), but the data are in the usual (lat, lon)
46+
# order, so reshape before comparing.
47+
data = data.reshape(NLATS, NLONS)
48+
assert_allclose([data[0, 0], data[NLATS//2, NLONS//2], data[-1, -1],
49+
data.min(), data.max()],
50+
[v00, vmid, vlast, vmin, vmax], rtol=1.e-5, atol=1.e-5)
51+
52+
def runTest(self):
53+
"""read GRIB2 file, with format detected by magic number"""
54+
with netCDF4.Dataset(FILE_NAME) as ds:
55+
self.check(ds)
56+
# selecting the GRIB2 reader explicitly gives the same result
57+
with netCDF4.Dataset(FILE_NAME, extra_mode_flag=netCDF4.NEP_UDF_GRIB2) as ds:
58+
self.check(ds)
59+
60+
61+
if __name__ == '__main__':
62+
unittest.main()

0 commit comments

Comments
 (0)