Merge branch 'develop' into feature/convert-core-edge-to-plasma-ids

Author: maarten-ic

conftest.py

@@ -9,7 +9,6 @@
 import functools
 import logging
 import os
-import sys
 from copy import deepcopy
 from pathlib import Path
 
@@ -22,7 +21,6 @@
 import pytest
 from packaging.version import Version
 
-from imas.backends.imas_core.imas_interface import has_imas as _has_imas
 from imas.backends.imas_core.imas_interface import ll_interface, lowlevel
 from imas.dd_zip import dd_etree, dd_xml_versions, latest_dd_version
 from imas.ids_defs import (
@@ -39,17 +37,7 @@
 
 os.environ["IMAS_AL_DISABLE_VALIDATE"] = "1"
 
-
-try:
-    import imas  # noqa
-except ImportError:
-
-    class SkipOnIMASAccess:
-        def __getattr__(self, attr):
-            pytest.skip("This test requires the `imas` HLI, which is not available.")
-
-    # Any test that tries to access an attribute from the `imas` package will be skipped
-    sys.modules["imas"] = SkipOnIMASAccess()
+import imas  # noqa
 
 
 def pytest_addoption(parser):
@@ -78,7 +66,6 @@ def pytest_addoption(parser):
     if "not available" in str(iex.message):
         _BACKENDS.pop("mdsplus")
 
-
 try:
     import pytest_xdist
 except ImportError:
@@ -91,28 +78,11 @@ def worker_id():
 @pytest.fixture(params=_BACKENDS)
 def backend(pytestconfig: pytest.Config, request: pytest.FixtureRequest):
     backends_provided = any(map(pytestconfig.getoption, _BACKENDS))
-    if not _has_imas:
-        if backends_provided:
-            raise RuntimeError(
-                "Explicit backends are provided, but IMAS is not available."
-            )
-        pytest.skip("No IMAS available, skip tests using a backend")
     if backends_provided and not pytestconfig.getoption(request.param):
         pytest.skip(f"Tests for {request.param} backend are skipped.")
     return _BACKENDS[request.param]
 
 
-@pytest.fixture()
-def has_imas():
-    return _has_imas
-
-
-@pytest.fixture()
-def requires_imas():
-    if not _has_imas:
-        pytest.skip("No IMAS available")
-
-
 def pytest_generate_tests(metafunc):
     if "ids_name" in metafunc.fixturenames:
         if metafunc.config.getoption("ids"):
@@ -214,7 +184,7 @@ def wrapper(*args, **kwargs):
 
 
 @pytest.fixture
-def log_lowlevel_calls(monkeypatch, requires_imas):
+def log_lowlevel_calls(monkeypatch):
     """Debugging fixture to log calls to the imas lowlevel module."""
     for al_function in dir(lowlevel):
         if al_function.startswith("ual_") or al_function.startswith("al"):

imas/backends/db_entry_impl.py

@@ -78,7 +78,7 @@ def get(
         destination: IDSToplevel,
         lazy: bool,
         nbc_map: Optional[NBCPathMap],
-    ) -> None:
+    ) -> IDSToplevel:
         """Implement DBEntry.get/get_slice/get_sample. Load data from the data source.
 
         Args:

imas/backends/imas_core/db_entry_al.py

@@ -38,7 +38,7 @@
 
 from .al_context import ALContext, LazyALContext
 from .db_entry_helpers import delete_children, get_children, put_children
-from .imas_interface import LLInterfaceError, has_imas, ll_interface
+from .imas_interface import LLInterfaceError, ll_interface
 from .mdsplus_model import mdsplus_model_dir
 from .uda_support import extract_idsdef, get_dd_version_from_idsdef_xml
 
@@ -52,14 +52,6 @@
 logger = logging.getLogger(__name__)
 
 
-def require_imas_available():
-    if not has_imas:
-        raise RuntimeError(
-            "The IMAS Core library is not available. Please install 'imas_core', "
-            "or load a supported IMAS module if you use an HPC environment."
-        )
-
-
 class ALDBEntryImpl(DBEntryImpl):
     """DBEntry implementation using imas_core as a backend."""
 
@@ -86,7 +78,6 @@ def __init__(self, uri: str, mode: int, factory: IDSFactory):
 
     @classmethod
     def from_uri(cls, uri: str, mode: str, factory: IDSFactory) -> "ALDBEntryImpl":
-        require_imas_available()
         if mode not in _OPEN_MODES:
             modes = list(_OPEN_MODES)
             raise ValueError(f"Unknown mode {mode!r}, was expecting any of {modes}")
@@ -105,8 +96,6 @@ def from_pulse_run(
         options: Any,
         factory: IDSFactory,
     ) -> "ALDBEntryImpl":
-        # Raise an error if imas is not available
-        require_imas_available()
 
         # Set defaults
         user_name = user_name or getpass.getuser()

imas/backends/imas_core/imas_interface.py

@@ -12,30 +12,17 @@
 
 from packaging.version import Version
 
-logger = logging.getLogger(__name__)
+# Import the Access Layer module
+# First try to import imas_core, which is available since AL 5.2
+from imas_core import _al_lowlevel as lowlevel
+from imas_core import imasdef  # noqa: F401
 
+logger = logging.getLogger(__name__)
 
-# Import the Access Layer module
-has_imas = True
-try:
-    # First try to import imas_core, which is available since AL 5.2
-    from imas_core import _al_lowlevel as lowlevel
-    from imas_core import imasdef
-
-    # Enable throwing exceptions from the _al_lowlevel interface
-    enable_exceptions = getattr(lowlevel, "imas_core_config_enable_exceptions", None)
-    if enable_exceptions:
-        enable_exceptions()
-
-except ImportError as exc:
-    imas = None
-    has_imas = False
-    imasdef = None
-    lowlevel = None
-    logger.warning(
-        "Could not import 'imas_core': %s. Some functionality is not available.",
-        exc,
-    )
+# Enable throwing exceptions from the _al_lowlevel interface
+enable_exceptions = getattr(lowlevel, "imas_core_config_enable_exceptions", None)
+if enable_exceptions:
+    enable_exceptions()
 
 
 class LLInterfaceError(RuntimeError):

imas/backends/netcdf/nc2ids.py

@@ -157,6 +157,8 @@ def run(self, lazy: bool) -> None:
                     for index, node in indexed_tree_iter(self.ids, target_metadata):
                         value = data[index]
                         if value != getattr(var, "_FillValue", None):
+                            if isinstance(value, np.generic):
+                                value = value.item()
                             # NOTE: bypassing IDSPrimitive.value.setter logic
                             node._IDSPrimitive__value = value
 
@@ -166,10 +168,16 @@ def run(self, lazy: bool) -> None:
                 # here, we'll let IDSPrimitive.value.setter take care of it:
                 self.ids[target_metadata.path].value = data
 
-            else:
+            # We need to unpack 0D ints, floats and complex numbers. For better
+            # performance this check is done outside the for-loop:
+            elif metadata.ndim or metadata.data_type is IDSDataType.STR:
                 for index, node in indexed_tree_iter(self.ids, target_metadata):
                     # NOTE: bypassing IDSPrimitive.value.setter logic
                     node._IDSPrimitive__value = data[index]
+            else:
+                for index, node in indexed_tree_iter(self.ids, target_metadata):
+                    # NOTE: bypassing IDSPrimitive.value.setter logic
+                    node._IDSPrimitive__value = data[index].item()  # Unpack 0D value
 
     def validate_variables(self) -> None:
         """Validate that all variables in the netCDF Group exist and match the DD."""
@@ -365,7 +373,7 @@ def get_child(self, child):
                 value = var[self.index]
 
             if value is not None:
-                if isinstance(value, np.ndarray):
+                if isinstance(value, (np.ndarray, np.generic)):
                     if value.ndim == 0:  # Unpack 0D numpy arrays:
                         value = value.item()
                     else:

imas/db_entry.py

@@ -160,7 +160,7 @@ def __init__(
                 legacy = True
             except TypeError as exc2:
                 raise TypeError(
-                    f"Incorrect arguments to {__class__.__name__}.__init__(): "
+                    "Incorrect arguments to DBEntry.__init__(): "
                     f"{exc1.args[0]}, {exc2.args[0]}"
                 ) from None
 
@@ -561,7 +561,7 @@ def _get(
             raise RuntimeError("Database entry is not open.")
         if lazy and destination:
             raise ValueError("Cannot supply a destination IDS when lazy loading.")
-        if not self._ids_factory.exists(ids_name) and autoconvert:
+        if autoconvert and not self._ids_factory.exists(ids_name):
             raise IDSNameError(ids_name, self._ids_factory)
 
         # Note: this will raise an exception when the ids/occurrence is not filled:
@@ -577,7 +577,7 @@ def _get(
                 ids_name,
                 occurrence,
             )
-        elif dd_version != self.dd_version and dd_version not in dd_xml_versions():
+        elif dd_version not in dd_xml_versions() and dd_version != self.dd_version:
             # We don't know the DD version that this IDS was written with
             if ignore_unknown_dd_version:
                 # User chooses to ignore this problem, load as if it was stored with

imas/exception.py

@@ -20,10 +20,8 @@
 
 
 # Expose ALException, which may be thrown by the lowlevel
-if _imas_interface.has_imas:
-    ALException = _imas_interface.lowlevel.ALException
-else:
-    ALException = None
+
+ALException = _imas_interface.lowlevel.ALException
 
 
 class IDSNameError(ValueError):

imas/ids_defs.py

@@ -86,86 +86,46 @@
     Identifier for the default serialization protocol.
 """
 
-import functools
 import logging
 
-from imas.backends.imas_core.imas_interface import has_imas, imasdef
+from imas.backends.imas_core.imas_interface import imasdef
 
 logger = logging.getLogger(__name__)
 
 
-if has_imas:
-    ASCII_BACKEND = imasdef.ASCII_BACKEND
-    CHAR_DATA = imasdef.CHAR_DATA
-    CLOSE_PULSE = imasdef.CLOSE_PULSE
-    CLOSEST_INTERP = imasdef.CLOSEST_INTERP
-    CREATE_PULSE = imasdef.CREATE_PULSE
-    DOUBLE_DATA = imasdef.DOUBLE_DATA
-    COMPLEX_DATA = imasdef.COMPLEX_DATA
-    EMPTY_COMPLEX = imasdef.EMPTY_COMPLEX
-    EMPTY_FLOAT = imasdef.EMPTY_FLOAT
-    EMPTY_INT = imasdef.EMPTY_INT
-    ERASE_PULSE = imasdef.ERASE_PULSE
-    FORCE_CREATE_PULSE = imasdef.FORCE_CREATE_PULSE
-    FORCE_OPEN_PULSE = imasdef.FORCE_OPEN_PULSE
-    HDF5_BACKEND = imasdef.HDF5_BACKEND
-    IDS_TIME_MODE_HETEROGENEOUS = imasdef.IDS_TIME_MODE_HETEROGENEOUS
-    IDS_TIME_MODE_HOMOGENEOUS = imasdef.IDS_TIME_MODE_HOMOGENEOUS
-    IDS_TIME_MODE_INDEPENDENT = imasdef.IDS_TIME_MODE_INDEPENDENT
-    IDS_TIME_MODE_UNKNOWN = imasdef.IDS_TIME_MODE_UNKNOWN
-    IDS_TIME_MODES = imasdef.IDS_TIME_MODES
-    INTEGER_DATA = imasdef.INTEGER_DATA
-    LINEAR_INTERP = imasdef.LINEAR_INTERP
-    MDSPLUS_BACKEND = imasdef.MDSPLUS_BACKEND
-    MEMORY_BACKEND = imasdef.MEMORY_BACKEND
-    NODE_TYPE_STRUCTURE = imasdef.NODE_TYPE_STRUCTURE
-    OPEN_PULSE = imasdef.OPEN_PULSE
-    PREVIOUS_INTERP = imasdef.PREVIOUS_INTERP
-    READ_OP = imasdef.READ_OP
-    UDA_BACKEND = imasdef.UDA_BACKEND
-    UNDEFINED_INTERP = imasdef.UNDEFINED_INTERP
-    UNDEFINED_TIME = imasdef.UNDEFINED_TIME
-    WRITE_OP = imasdef.WRITE_OP
-    ASCII_SERIALIZER_PROTOCOL = getattr(imasdef, "ASCII_SERIALIZER_PROTOCOL", 60)
-    FLEXBUFFERS_SERIALIZER_PROTOCOL = getattr(
-        imasdef, "FLEXBUFFERS_SERIALIZER_PROTOCOL", None
-    )
-    DEFAULT_SERIALIZER_PROTOCOL = getattr(imasdef, "DEFAULT_SERIALIZER_PROTOCOL", 60)
-
-else:
-    # Preset some constants which are used elsewhere
-    # this is a bit ugly, perhaps reuse the list of imports from above?
-    # it seems no problem to use None, since the use of the values should not
-    # be allowed, they are only used in operations which use the backend,
-    # which we (should) gate
-    ASCII_BACKEND = CHAR_DATA = CLOSE_PULSE = CLOSEST_INTERP = DOUBLE_DATA = None
-    FORCE_OPEN_PULSE = CREATE_PULSE = ERASE_PULSE = None
-    COMPLEX_DATA = FORCE_CREATE_PULSE = HDF5_BACKEND = None
-    INTEGER_DATA = LINEAR_INTERP = MDSPLUS_BACKEND = MEMORY_BACKEND = None
-    NODE_TYPE_STRUCTURE = OPEN_PULSE = PREVIOUS_INTERP = READ_OP = None
-    UDA_BACKEND = UNDEFINED_INTERP = UNDEFINED_TIME = WRITE_OP = None
-    # These constants are also useful when not working with the AL
-    EMPTY_FLOAT = -9e40
-    EMPTY_INT = -999_999_999
-    EMPTY_COMPLEX = complex(EMPTY_FLOAT, EMPTY_FLOAT)
-    IDS_TIME_MODE_UNKNOWN = EMPTY_INT
-    IDS_TIME_MODE_HETEROGENEOUS = 0
-    IDS_TIME_MODE_HOMOGENEOUS = 1
-    IDS_TIME_MODE_INDEPENDENT = 2
-    IDS_TIME_MODES = [0, 1, 2]
-    ASCII_SERIALIZER_PROTOCOL = 60
-    FLEXBUFFERS_SERIALIZER_PROTOCOL = None
-    DEFAULT_SERIALIZER_PROTOCOL = 60
-
-
-def needs_imas(func):
-    if has_imas:
-        return func
-
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        raise RuntimeError(
-            f"Function {func.__name__} requires IMAS, but IMAS is not available."
-        )
-
-    return wrapper
+ASCII_BACKEND = imasdef.ASCII_BACKEND
+CHAR_DATA = imasdef.CHAR_DATA
+CLOSE_PULSE = imasdef.CLOSE_PULSE
+CLOSEST_INTERP = imasdef.CLOSEST_INTERP
+CREATE_PULSE = imasdef.CREATE_PULSE
+DOUBLE_DATA = imasdef.DOUBLE_DATA
+COMPLEX_DATA = imasdef.COMPLEX_DATA
+EMPTY_COMPLEX = imasdef.EMPTY_COMPLEX
+EMPTY_FLOAT = imasdef.EMPTY_FLOAT
+EMPTY_INT = imasdef.EMPTY_INT
+ERASE_PULSE = imasdef.ERASE_PULSE
+FORCE_CREATE_PULSE = imasdef.FORCE_CREATE_PULSE
+FORCE_OPEN_PULSE = imasdef.FORCE_OPEN_PULSE
+HDF5_BACKEND = imasdef.HDF5_BACKEND
+IDS_TIME_MODE_HETEROGENEOUS = imasdef.IDS_TIME_MODE_HETEROGENEOUS
+IDS_TIME_MODE_HOMOGENEOUS = imasdef.IDS_TIME_MODE_HOMOGENEOUS
+IDS_TIME_MODE_INDEPENDENT = imasdef.IDS_TIME_MODE_INDEPENDENT
+IDS_TIME_MODE_UNKNOWN = imasdef.IDS_TIME_MODE_UNKNOWN
+IDS_TIME_MODES = imasdef.IDS_TIME_MODES
+INTEGER_DATA = imasdef.INTEGER_DATA
+LINEAR_INTERP = imasdef.LINEAR_INTERP
+MDSPLUS_BACKEND = imasdef.MDSPLUS_BACKEND
+MEMORY_BACKEND = imasdef.MEMORY_BACKEND
+NODE_TYPE_STRUCTURE = imasdef.NODE_TYPE_STRUCTURE
+OPEN_PULSE = imasdef.OPEN_PULSE
+PREVIOUS_INTERP = imasdef.PREVIOUS_INTERP
+READ_OP = imasdef.READ_OP
+UDA_BACKEND = imasdef.UDA_BACKEND
+UNDEFINED_INTERP = imasdef.UNDEFINED_INTERP
+UNDEFINED_TIME = imasdef.UNDEFINED_TIME
+WRITE_OP = imasdef.WRITE_OP
+ASCII_SERIALIZER_PROTOCOL = getattr(imasdef, "ASCII_SERIALIZER_PROTOCOL", 60)
+FLEXBUFFERS_SERIALIZER_PROTOCOL = getattr(
+    imasdef, "FLEXBUFFERS_SERIALIZER_PROTOCOL", None
+)
+DEFAULT_SERIALIZER_PROTOCOL = getattr(imasdef, "DEFAULT_SERIALIZER_PROTOCOL", 60)