A curated, citable collection of electron microscopy datasets — '
- 'a couple of lines of Python from your analysis. Search below, then copy the '
- 'snippet to load one with em_database.data.<Name>().
'
- '
'
+ "
EM-Database
"
+ "
A curated, citable collection of electron microscopy datasets — "
+ "a couple of lines of Python from your analysis. Search below, then copy the "
+ "snippet to load one with em_database.data.<Name>().
Fill in the metadata; the YAML builds live on the right. '
- '“Open a Pull Request” sends you to GitHub with the new file '
- 'pre-filled — commit it to a branch there and GitHub opens the PR.
'
- '
'
+ "
Fill in the metadata; the YAML builds live on the right. "
+ "“Open a Pull Request” sends you to GitHub with the new file "
+ "pre-filled — commit it to a branch there and GitHub opens the PR.
"
+ "
"
'
'
''
+ ""
'
"
'
Requires a GitHub account. The button opens GitHub’s '
- '“create new file” page pre-filled at '
- 'em_database/datasets/<Name>.yaml; if you cannot push to the '
- 'repo, GitHub forks it for you and lets you propose the change. Fields marked '
+ "“create new file” page pre-filled at "
+ "em_database/datasets/<Name>.yaml; if you cannot push to the "
+ "repo, GitHub forks it for you and lets you propose the change. Fields marked "
'* are required.
'
- ''
- ''
- ''
+ ""
+ ""
+ ""
)
js = _ADD_DATASET_JS.replace("__REPO__", _REPO).replace("__BRANCH__", _BRANCH)
scripts = ""
- return _app_page("Add Dataset · EM-Database", body, active="Add Dataset",
- extra_css=_FORM_CSS, scripts=scripts)
+ return _app_page(
+ "Add Dataset · EM-Database",
+ body,
+ active="Add Dataset",
+ extra_css=_FORM_CSS,
+ scripts=scripts,
+ )
_ADD_DATASET_JS = r"""
@@ -1145,4 +1215,4 @@ def generate_add_dataset_html() -> str:
(out / "all_data.html").write_text(generate_all_data_html(), encoding="utf-8")
(out / "add_dataset.html").write_text(generate_add_dataset_html(), encoding="utf-8")
(out / "datasets_browser.html").write_text(generate_browser_html(), encoding="utf-8")
- print("Wrote preview pages to", out.resolve())
\ No newline at end of file
+ print("Wrote preview pages to", out.resolve())
diff --git a/em_database/_create_stubs.py b/em_database/_create_stubs.py
index fd25fac..6c51be7 100644
--- a/em_database/_create_stubs.py
+++ b/em_database/_create_stubs.py
@@ -1,21 +1,24 @@
-import os
-import yaml
from pathlib import Path
+
+import yaml
+
+
# on start up set data dir if not already set
def build_docstring(dataset_dict) -> str:
- """ Build a docstring for the dataset from its metadata. """
- doc = f""
+ """Build a docstring for the dataset from its metadata."""
+ doc = ""
if dataset_dict.get("description"):
- doc += f"{dataset_dict["description"]}\n\n"
+ doc += f"{dataset_dict['description']}\n\n"
if dataset_dict.get("doi"):
- doc += f" DOI: {dataset_dict["doi"]}\n\n"
+ doc += f" DOI: {dataset_dict['doi']}\n\n"
if dataset_dict.get("license"):
- doc += f" License: {dataset_dict["license"]}\n\n"
+ doc += f" License: {dataset_dict['license']}\n\n"
doc += " You can download this dataset here:\n"
- doc += f" {dataset_dict["source"]}\n\n"
+ doc += f" {dataset_dict['source']}\n\n"
return doc
+
def generate_pyi_stub():
"""Generate a .pyi stub file for IDE autocomplete support."""
stub_lines = [
@@ -27,37 +30,32 @@ def generate_pyi_stub():
# Collect all dataset classes
dataset_classes = []
- for root, dirs, files in os.walk(os.path.join(os.path.dirname(__file__), "datasets")):
- for file in files:
- if file.endswith(".yaml") or file.endswith(".yml"):
- dataset_path = os.path.join(root, file)
- with open(dataset_path, 'r') as f:
- data_dict_yaml = yaml.safe_load(f)
- for name in data_dict_yaml:
- data_dict = data_dict_yaml[name]
- class_name = name.replace(' ', '_').replace('-', '_')
- description = build_docstring(data_dict)
-
- # Add class definition to stub
- stub_lines.append(f"class {class_name}(DownloadableDataset):")
- stub_lines.append(f' """')
- stub_lines.append(f" {name}")
- if description:
- stub_lines.append(f" ")
- stub_lines.append(f" {description}")
- stub_lines.append(f' """')
- stub_lines.append(" ...")
- stub_lines.append("")
-
- dataset_classes.append(class_name)
-
- # Add __all__ to stub
- stub_lines.append(f"__all__ = __all__ + {dataset_classes }")
+ for dataset_path in sorted((Path(__file__).parent / "datasets").rglob("*.y*ml")):
+ data_dict_yaml = yaml.safe_load(dataset_path.read_text(encoding="utf-8"))
+ for name in data_dict_yaml:
+ data_dict = data_dict_yaml[name]
+ class_name = name.replace(" ", "_").replace("-", "_")
+ description = build_docstring(data_dict)
+
+ stub_lines.append(f"class {class_name}(DownloadableDataset):")
+ stub_lines.append(' """')
+ stub_lines.append(f" {name}")
+ if description:
+ stub_lines.append(" ")
+ stub_lines.append(f" {description}")
+ stub_lines.append(' """')
+ stub_lines.append(" ...")
+ stub_lines.append("")
+
+ dataset_classes.append(class_name)
+
+ stub_lines.append(f"__all__ = {dataset_classes}")
# Write stub file
- stub_path = Path(__file__).parent / "data"/"__init__.pyi"
- with open(stub_path, 'w') as f:
- f.write('\n'.join(stub_lines))
+ stub_path = Path(__file__).parent / "data" / "__init__.pyi"
+ with open(stub_path, "w") as f:
+ f.write("\n".join(stub_lines))
+
if __name__ == "__main__":
generate_pyi_stub()
diff --git a/em_database/catalogue.py b/em_database/catalogue.py
index 494492e..d86740f 100644
--- a/em_database/catalogue.py
+++ b/em_database/catalogue.py
@@ -5,35 +5,39 @@
marked downloaded or not, each carrying the metadata a user hovers to read. It
downloads nothing and opens no files, so it is cheap enough to rebuild on every
render (the one thing that changes underfoot is which files are on disk, which
-is a single ``os.path.exists`` per dataset).
+is a single ``Path.exists`` per dataset).
"""
+
from __future__ import annotations
import inspect
-from typing import Any, Optional
+from pathlib import Path
+from em_database.downloadable_dataset import DownloadableDataset
# Techniques in the order the browser should show them - the modalities the
# collection is built around first, then anything else alphabetically.
TECHNIQUE_ORDER = ("4D-STEM", "EELS", "EDS", "EBSD", "STEM", "In-situ TEM", "Cryo-EM")
-def datasets() -> list[tuple[str, Any]]:
+def datasets() -> list[tuple[str, DownloadableDataset]]:
"""``(name, dataset)`` for every dataset ``em_database.data`` exposes.
Filtered by ``issubclass`` so the base class and incidental imports in the
module namespace stay out; sorted by name for a stable order.
"""
import em_database.data as data
- from em_database.downloadable_dataset import DownloadableDataset
- out: list[tuple[str, Any]] = []
+ out: list[tuple[str, DownloadableDataset]] = []
for name in getattr(data, "__all__", None) or dir(data):
if name.startswith("_"):
continue
obj = getattr(data, name, None)
- if (not inspect.isclass(obj) or obj is DownloadableDataset
- or not issubclass(obj, DownloadableDataset)):
+ if (
+ not inspect.isclass(obj)
+ or obj is DownloadableDataset
+ or not issubclass(obj, DownloadableDataset)
+ ):
continue
try:
out.append((name, obj()))
@@ -42,12 +46,14 @@ def datasets() -> list[tuple[str, Any]]:
return sorted(out, key=lambda kv: kv[0].lower())
-def resolve(name: str):
+def resolve(name: str) -> DownloadableDataset | None:
"""The dataset instance for a catalogue name, or ``None``."""
import em_database.data as data
obj = getattr(data, str(name), None)
- return obj() if inspect.isclass(obj) else None
+ if not inspect.isclass(obj) or not issubclass(obj, DownloadableDataset):
+ return None
+ return obj()
def _technique(ds) -> str:
@@ -59,7 +65,7 @@ def _join(*parts) -> str:
return " ".join(str(p).strip() for p in parts if p and str(p).strip())
-def _declared_shape(ds) -> Optional[str]:
+def _declared_shape(ds) -> str | None:
"""The shape em-database declares in the dataset's YAML, if it does.
Only declared shapes are used - reading it out of a downloaded file would
@@ -92,7 +98,18 @@ def _authors(md) -> tuple[list[str], list[str]]:
return [str(a) for a in (authors or [])], []
-def entry(name: str, ds) -> dict:
+def _location(path: Path | None) -> str | None:
+ """Which data directory a downloaded file came from: "shared" or "user"."""
+ if path is None:
+ return None
+ from em_database import config
+
+ parent = path.resolve().parent
+ shared = {d.resolve() for d in config.shared_data_dirs()}
+ return "shared" if parent in shared else "user"
+
+
+def entry(name: str, ds: DownloadableDataset) -> dict:
"""One catalogue row - everything the browser draws for a dataset."""
md = getattr(ds, "metadata", None) or {}
try:
@@ -105,8 +122,9 @@ def entry(name: str, ds) -> dict:
"technique": _technique(ds),
"size": str(getattr(ds, "data_size", "") or ""),
"shape": _declared_shape(ds),
- "downloaded": bool(path),
- "path": path or "",
+ "downloaded": path is not None,
+ "location": _location(path),
+ "path": str(path) if path else "",
"description": str(getattr(ds, "description", "") or ""),
"detector": _join(getattr(ds, "detector_manufacturer", ""), getattr(ds, "detector", "")),
"microscope": _join(md.get("microscope_vendor"), md.get("microscope_model")),
@@ -122,10 +140,19 @@ def entry(name: str, ds) -> dict:
# "Carter Francis" (an author) or "Direct Electron" (an affiliation) finds
# every dataset it touches - not just the name.
searchable = [
- name, row["technique"], row["description"], row["detector"],
- row["microscope"], row["voltage"], row["license"], row["doi"],
- row["file"], row["shape"] or "", " ".join(row["tags"]),
- " ".join(names), " ".join(affiliations),
+ name,
+ row["technique"],
+ row["description"],
+ row["detector"],
+ row["microscope"],
+ row["voltage"],
+ row["license"],
+ row["doi"],
+ row["file"],
+ row["shape"] or "",
+ " ".join(row["tags"]),
+ " ".join(names),
+ " ".join(affiliations),
]
row["search"] = " ".join(str(s) for s in searchable if s).lower()
return row
diff --git a/em_database/config.py b/em_database/config.py
index 1fa7bb3..2a03346 100644
--- a/em_database/config.py
+++ b/em_database/config.py
@@ -24,6 +24,7 @@
environment variable (an ``os.pathsep``-separated list), or a ``shared_data_dirs``
list in the user's settings.
"""
+
from __future__ import annotations
import os
@@ -32,19 +33,20 @@
import yaml
-def _default_data_dir() -> str:
- return os.path.join(os.path.expanduser("~"), "em_database")
+def _default_data_dir() -> Path:
+ return Path.home() / "em_database"
-#: Built-in defaults - the fallback when nothing is configured.
+#: Built-in defaults - the fallback when nothing is configured. Values here are
+#: the serialized (string) form, so they compare equal to what the YAML holds.
DEFAULTS: dict = {
- "data_dir": _default_data_dir(),
+ "data_dir": str(_default_data_dir()),
}
def config_dir() -> Path:
"""The ``~/.em_database`` folder that holds the settings file."""
- return Path(os.path.expanduser("~")) / ".em_database"
+ return Path.home() / ".em_database"
def config_path() -> Path:
@@ -155,6 +157,7 @@ def reset(self, key: str | None = None) -> None:
def widget(self):
"""Return an interactive widget for editing the settings (Jupyter)."""
from em_database.widget import settings_widget
+
return settings_widget()
def _repr_mimebundle_(self, **kwargs):
@@ -162,6 +165,7 @@ def _repr_mimebundle_(self, **kwargs):
plain dict repr if anywidget is not installed)."""
try:
from em_database.widget import settings_widget
+
widget = settings_widget()
except Exception:
return {"text/plain": repr(dict(self))}
@@ -173,12 +177,12 @@ def _repr_mimebundle_(self, **kwargs):
_seed(settings)
-def data_dir() -> str:
+def data_dir() -> Path:
"""The user's data directory - where downloads are written."""
- return str(settings.get("data_dir") or _default_data_dir())
+ return Path(settings.get("data_dir") or _default_data_dir())
-def shared_data_dirs() -> list[str]:
+def shared_data_dirs() -> list[Path]:
"""System-wide / shared data locations, checked before the user's dir.
Order: ``EM_DATABASE_SHARED_DIR`` (an ``os.pathsep`` list), then the user's
@@ -195,15 +199,15 @@ def shared_data_dirs() -> list[str]:
dirs.append(str(system["data_dir"]))
dirs += [str(d) for d in (system.get("shared_data_dirs") or [])]
seen: set[str] = set()
- unique: list[str] = []
+ unique: list[Path] = []
for d in dirs:
if d not in seen:
seen.add(d)
- unique.append(d)
+ unique.append(Path(d))
return unique
-def data_search_dirs() -> list[str]:
+def data_search_dirs() -> list[Path]:
"""Everywhere to look for an existing dataset: shared/system dirs first,
then the user's data directory."""
dirs = shared_data_dirs()
diff --git a/em_database/data/__init__.py b/em_database/data/__init__.py
index f0e5ed5..7d60fc7 100644
--- a/em_database/data/__init__.py
+++ b/em_database/data/__init__.py
@@ -1,36 +1,23 @@
-""" Auto-generated dataset classes from YAML Files for downloading data."""
-import os
+"""Auto-generated dataset classes from YAML Files for downloading data."""
+
from pathlib import Path
import yaml
-from em_database.downloadable_dataset import DownloadableDataset
+
from em_database._create_stubs import build_docstring
+from em_database.downloadable_dataset import DownloadableDataset
-# Map all the datasets in the "datasets" folder
-# recursively travel down
__all__ = []
datasets_path = Path(__file__).parent.parent / "datasets"
-for root, dirs, files in os.walk(datasets_path):
- for file in files:
- if file.endswith(".yaml") or file.endswith(".yml"):
- dataset_path = os.path.join(root, file)
- with open(dataset_path, 'r') as f:
- data_dict_yaml = yaml.safe_load(f)
- for name in data_dict_yaml:
- class_name = name.replace(' ', '_').replace('-', '_')
- data_dict = data_dict_yaml[name]
- def _make_init(data):
- def __init__(self):
- super(self.__class__, self).__init__(**data)
-
- return __init__
-
-
- _new_class = type(class_name,
- (DownloadableDataset,),
- {"__init__": _make_init(data_dict),
- "__doc__": build_docstring(data_dict)})
-
- # Add to module globals and __all__
- globals()[class_name] = _new_class
- __all__.append(class_name)
\ No newline at end of file
+for dataset_path in sorted(datasets_path.rglob("*.y*ml")):
+ data_dict_yaml = yaml.safe_load(dataset_path.read_text(encoding="utf-8"))
+ for name in data_dict_yaml:
+ class_name = name.replace(" ", "_").replace("-", "_")
+ data_dict = data_dict_yaml[name]
+ _new_class = type(
+ class_name,
+ (DownloadableDataset,),
+ {"_spec": data_dict, "__doc__": build_docstring(data_dict)},
+ )
+ globals()[class_name] = _new_class
+ __all__.append(class_name)
diff --git a/em_database/data/__init__.pyi b/em_database/data/__init__.pyi
index c28ba0e..0b0698b 100644
--- a/em_database/data/__init__.pyi
+++ b/em_database/data/__init__.pyi
@@ -1,137 +1,129 @@
# Auto-generated stub file for em_database
from em_database.downloadable_dataset import DownloadableDataset
-class AlNanocrystals(DownloadableDataset):
+class LSMOLineScan(DownloadableDataset):
"""
- AlNanocrystals
-
- A 4D STEM dataset of Al nanocrystals on a carbon support.
+ LSMOLineScan
- License: CC-BY-4.0
+ A core-loss EELS line scan through a La(0.7)Sr(0.3)MnO3 thin film in which part of the film was deliberately given a very long electron beam exposure, inducing oxygen vacancies. Used by the eXSpy fine structure tutorial - the O-K and Mn-L2,3 fine structure changes measurably between the damaged and undamaged regions. 40 probe positions at 3.219 nm with 586 energy channels covering 428.5-721 eV at 0.5 eV dispersion. Acquired on a JEOL ARM200cF with a Gatan Quantum ER in DualEELS mode at 80 kV, 27.42 mrad convergence and 33.19 mrad collection angle. The matching low-loss spectrum is LSMOLineScanLowLoss.
+
+ License: Unspecified
You can download this dataset here:
- https://zenodo.org/records/15490547/files
+ https://raw.githubusercontent.com/hyperspy/exspy-demos/927d1f21b3b8aba4e2e622c2e621d3d9d5542d1c/EELS/datasets
"""
+
...
-class AmorphousFilm4nm4DSTEM(DownloadableDataset):
+class PdNiPGlass(DownloadableDataset):
"""
- AmorphousFilm4nm4DSTEM
-
- A 4D-STEM dataset of a 4 nm amorphous thin film acquired with a 2.5 mrad probe on a Direct Electron CeleritasXS at 49000 fps. 256 x 256 probe positions (the central quarter of a 1024 x 1024 scan) of 128 x 128 pixel diffraction patterns. Both real and reciprocal space are calibrated - 0.12325 nm per scan step and 0.12453 1/nm per detector pixel, centred on the direct beam. Suitable for fluctuation electron microscopy and angular correlation analysis. Gain- and dark-corrected intensities were divided by 2 and rounded to uint16; multiply by 2 to recover ADU (the detector records 300 ADU per electron).
+ PdNiPGlass
- DOI: 10.5281/zenodo.21632101
+ A 4D STEM dataset of PdNiP metallic glass thin film.
License: CC-BY-4.0
You can download this dataset here:
- https://zenodo.org/records/21632101/files
+ https://zenodo.org/records/15490547/files
"""
+
...
-class ApoferritinApollo15eps(DownloadableDataset):
+class FeAlStripes(DownloadableDataset):
"""
- ApoferritinApollo15eps
-
- A single cryo-EM movie of apoferritin from one stage position, collected at a 15 e-/pix/s dose rate on a Direct Electron Apollo. 76 unaligned, dark-subtracted counted super-resolution frames of 8192 x 8192 pixels at 0.2995 Angstrom per pixel, totalling 56.96 e-/Angstrom^2 (0.7495 e-/Angstrom^2 per frame). The super-resolution gain reference needed for frame correction is embedded in the file at metadata.Acquisition_instrument.TEM.Detector.gain_reference. Repackaged from EMPIAR-11254; see Peng et al., J Struct Biol X 7 (2022) 100080.
+ FeAlStripes
- DOI: 10.5281/zenodo.21632101
+ A 4D STEM dataset with FeAl stripes exhibiting magnetic stripes. The dataset can be used to study the correlation between structural and magnetic properties.
- License: CC0-1.0
+ License: CC-BY-4.0
You can download this dataset here:
- https://zenodo.org/records/21632101/files
+ https://zenodo.org/records/15490547/files
"""
+
...
-class BilayerWS2(DownloadableDataset):
+class InSituElectrochemGrowth(DownloadableDataset):
"""
- BilayerWS2
-
- small 4-D STEM dataset of a bilayer WS2. Each Diffraction pattern is only 8x8 pixels so the dataset is quite small although for simple non iterative ptychography 8x8 pixels should be sufficient.
+ InSituElectrochemGrowth
+
+ An in-situ electrochemistry TEM movie showing growth in a liquid cell, recorded at 300 kV on a Direct Electron DE-Artemis in hardware counting mode. 245 frames of 4096 x 4096 pixels - every 4th frame of a 977 frame movie - with a calibrated 0.45448 nm pixel size and a 0.26208 s interval between saved frames (1.86 x 1.86 micron field of view, 64 s of elapsed time).
+
+ DOI: 10.5281/zenodo.21632101
License: CC-BY-4.0
You can download this dataset here:
- https://zenodo.org/records/15490547/files
+ https://zenodo.org/records/21632101/files
"""
+
...
-class CuZnEELSMapping(DownloadableDataset):
+class ApoferritinApollo15eps(DownloadableDataset):
"""
- CuZnEELSMapping
-
- An EELS spectrum image of copper and zinc oxide deposited on carbon nanotubes, used by the eXSpy elemental mapping tutorial. 40 x 50 probe positions at 0.9214 nm with 162 energy channels covering 700-1988 eV at 8 eV dispersion, which resolves the Cu-L2,3 (~931 eV) and Zn-L2,3 (~1020 eV) edges. The Zn:Cu ratio is 3:1 and roughly 80 wt% of the sample is carbon, so the edges sit on a large plasmon background - a good test case for model-based background removal and overlapping-edge quantification. The simultaneously acquired survey image is available as CuZnHAADF. Note - the Sample.description field in the file reads "Ta2O5 25% TiO2 CSIRO 400C", which is stale metadata carried over from an unrelated acquisition.
-
- License: Unspecified
-
- You can download this dataset here:
- https://raw.githubusercontent.com/hyperspy/exspy-demos/927d1f21b3b8aba4e2e622c2e621d3d9d5542d1c/EELS/datasets
-
+ ApoferritinApollo15eps
- """
- ...
+ A single cryo-EM movie of apoferritin from one stage position, collected at a 15 e-/pix/s dose rate on a Direct Electron Apollo. 76 unaligned, dark-subtracted counted super-resolution frames of 8192 x 8192 pixels at 0.2995 Angstrom per pixel, totalling 56.96 e-/Angstrom^2 (0.7495 e-/Angstrom^2 per frame). The super-resolution gain reference needed for frame correction is embedded in the file at metadata.Acquisition_instrument.TEM.Detector.gain_reference. Repackaged from EMPIAR-11254; see Peng et al., J Struct Biol X 7 (2022) 100080.
-class CuZnHAADF(DownloadableDataset):
- """
- CuZnHAADF
-
- The HAADF survey image acquired simultaneously with the CuZnEELSMapping spectrum image - copper and zinc oxide on carbon nanotubes. 40 x 50 pixels at 0.9214 nm, uint16, on a JEOL ARM200F at 200 kV and 600000x. Pairs with CuZnEELSMapping for correlating elemental maps against the survey. Note - the Sample.description field in the file reads "Ta2O5 25% TiO2 CSIRO 400C", which is stale metadata carried over from an unrelated acquisition.
+ DOI: 10.5281/zenodo.21632101
- License: Unspecified
+ License: CC0-1.0
You can download this dataset here:
- https://raw.githubusercontent.com/hyperspy/exspy-demos/927d1f21b3b8aba4e2e622c2e621d3d9d5542d1c/EELS/datasets
+ https://zenodo.org/records/21632101/files
"""
+
...
-class FeAlStripes(DownloadableDataset):
+class AmorphousFilm4nm4DSTEM(DownloadableDataset):
"""
- FeAlStripes
-
- A 4D STEM dataset with FeAl stripes exhibiting magnetic stripes. The dataset can be used to study the correlation between structural and magnetic properties.
+ AmorphousFilm4nm4DSTEM
+
+ A 4D-STEM dataset of a 4 nm amorphous thin film acquired with a 2.5 mrad probe on a Direct Electron CeleritasXS at 49000 fps. 256 x 256 probe positions (the central quarter of a 1024 x 1024 scan) of 128 x 128 pixel diffraction patterns. Both real and reciprocal space are calibrated - 0.12325 nm per scan step and 0.12453 1/nm per detector pixel, centred on the direct beam. Suitable for fluctuation electron microscopy and angular correlation analysis. Gain- and dark-corrected intensities were divided by 2 and rounded to uint16; multiply by 2 to recover ADU (the detector records 300 ADU per electron).
+
+ DOI: 10.5281/zenodo.21632101
License: CC-BY-4.0
You can download this dataset here:
- https://zenodo.org/records/15490547/files
+ https://zenodo.org/records/21632101/files
"""
+
...
-class HREBSDStrainPatterns(DownloadableDataset):
+class SPEDAg(DownloadableDataset):
"""
- HREBSDStrainPatterns
-
- High-resolution EBSD patterns collected on a Direct Electron DE-Meridian, centred on a deformed region suitable for cross-correlation strain analysis. 32 x 32 probe positions at a 25 nm step, each pattern the sum of 256 counted, dark- and gain-corrected frames, binned 2 x 2 from 2048 x 2048 to 1024 x 1024 pixels. The pattern-plane geometry (detector distance, pattern centre, sample tilt) and the accelerating voltage were not recorded with the raw data, so the pattern axes are in detector pixels.
+ SPEDAg
- DOI: 10.5281/zenodo.21632101
+ A 4D STEM dataset of polycrystalline Ag including twins and grain boundaries.
License: CC-BY-4.0
You can download this dataset here:
- https://zenodo.org/records/21632101/files
+ https://zenodo.org/records/15490547/files
"""
+
...
-class InSituElectrochemGrowth(DownloadableDataset):
+class LayeredCuNb4DSTEM(DownloadableDataset):
"""
- InSituElectrochemGrowth
-
- An in-situ electrochemistry TEM movie showing growth in a liquid cell, recorded at 300 kV on a Direct Electron DE-Artemis in hardware counting mode. 245 frames of 4096 x 4096 pixels - every 4th frame of a 977 frame movie - with a calibrated 0.45448 nm pixel size and a 0.26208 s interval between saved frames (1.86 x 1.86 micron field of view, 64 s of elapsed time).
+ LayeredCuNb4DSTEM
+
+ A 4D-STEM dataset of a layered Cu/Nb nanolaminate, acquired with a nearly parallel 1.58 mrad probe on a Direct Electron CeleritasXS. 128 x 128 probe positions (the central quarter of a 512 x 512 raster scan) of 256 x 256 pixel diffraction patterns, each the sum of 32 camera frames at 25000 fps. Reciprocal space is calibrated at 0.0078768 1/Angstrom per pixel and centred on the direct beam; the detector half-width is 1.008 1/Angstrom (25.3 mrad at 200 kV). The real-space scan step was not recorded by the scan controller, so the navigation axes are in pixels. Gain- and dark-corrected intensities were divided by 2 and rounded to uint16; multiply by 2 to recover ADU.
DOI: 10.5281/zenodo.21632101
@@ -142,44 +134,45 @@ class InSituElectrochemGrowth(DownloadableDataset):
"""
+
...
-class LayeredCuNb4DSTEM(DownloadableDataset):
+class BilayerWS2(DownloadableDataset):
"""
- LayeredCuNb4DSTEM
-
- A 4D-STEM dataset of a layered Cu/Nb nanolaminate, acquired with a nearly parallel 1.58 mrad probe on a Direct Electron CeleritasXS. 128 x 128 probe positions (the central quarter of a 512 x 512 raster scan) of 256 x 256 pixel diffraction patterns, each the sum of 32 camera frames at 25000 fps. Reciprocal space is calibrated at 0.0078768 1/Angstrom per pixel and centred on the direct beam; the detector half-width is 1.008 1/Angstrom (25.3 mrad at 200 kV). The real-space scan step was not recorded by the scan controller, so the navigation axes are in pixels. Gain- and dark-corrected intensities were divided by 2 and rounded to uint16; multiply by 2 to recover ADU.
+ BilayerWS2
- DOI: 10.5281/zenodo.21632101
+ small 4-D STEM dataset of a bilayer WS2. Each Diffraction pattern is only 8x8 pixels so the dataset is quite small although for simple non iterative ptychography 8x8 pixels should be sufficient.
License: CC-BY-4.0
You can download this dataset here:
- https://zenodo.org/records/21632101/files
+ https://zenodo.org/records/15490547/files
"""
+
...
-class LSMOLineScan(DownloadableDataset):
+class NiEBSDLarge(DownloadableDataset):
"""
- LSMOLineScan
-
- A core-loss EELS line scan through a La(0.7)Sr(0.3)MnO3 thin film in which part of the film was deliberately given a very long electron beam exposure, inducing oxygen vacancies. Used by the eXSpy fine structure tutorial - the O-K and Mn-L2,3 fine structure changes measurably between the damaged and undamaged regions. 40 probe positions at 3.219 nm with 586 energy channels covering 428.5-721 eV at 0.5 eV dispersion. Acquired on a JEOL ARM200cF with a Gatan Quantum ER in DualEELS mode at 80 kV, 27.42 mrad convergence and 33.19 mrad collection angle. The matching low-loss spectrum is LSMOLineScanLowLoss.
+ NiEBSDLarge
- License: Unspecified
+ 4125 EBSD patterns in a (55, 75) navigation shape of (60, 60) pixels from nickel, acquired on a NORDIF UF-1100 detector
+
+ License: CC-BY-4.0
You can download this dataset here:
- https://raw.githubusercontent.com/hyperspy/exspy-demos/927d1f21b3b8aba4e2e622c2e621d3d9d5542d1c/EELS/datasets
+ https://raw.githubusercontent.com/pyxem/kikuchipy-data/bcab8f7a4ffdb86a97f14e2327a4813d3156a85e/nickel_ebsd_large/
"""
+
...
class LSMOLineScanLowLoss(DownloadableDataset):
"""
LSMOLineScanLowLoss
-
+
The low-loss half of the DualEELS pair for LSMOLineScan - a La(0.7)Sr(0.3)MnO3 thin film line scan through a beam-damaged, oxygen-deficient region. 40 probe positions at 3.219 nm with 1024 energy channels covering -50 to 461.5 eV at 0.5 eV dispersion, containing the zero-loss peak and plasmon region. Use it with the core-loss spectrum for relative thickness mapping and Fourier-ratio deconvolution before fine structure analysis. Acquired on a JEOL ARM200cF with a Gatan Quantum ER at 80 kV.
License: Unspecified
@@ -189,12 +182,13 @@ class LSMOLineScanLowLoss(DownloadableDataset):
"""
+
...
class LSMOSTOLineScan(DownloadableDataset):
"""
LSMOSTOLineScan
-
+
A core-loss EELS line scan across a La(0.7)Sr(0.3)MnO3 thin film grown on SrTiO3, used by the eXSpy perovskite oxide analysis tutorial. 10 probe positions at 3.152 nm with 512 energy channels covering 395-906 eV at 1 eV dispersion, which contains the Ti-L2,3, O-K, Mn-L2,3 and La-M4,5 edges - so a single line scan crosses the interface and shows the Ti signal give way to La and Mn. Acquired on a JEOL ARM200cF with a Gatan Quantum ER in DualEELS mode at 200 kV, 27.1 mrad convergence and 33.1 mrad collection angle. The matching low-loss spectrum needed for thickness correction and Fourier-ratio deconvolution is LSMOSTOLineScanLowLoss. Binned from the original acquisition to keep the file small.
License: Unspecified
@@ -204,12 +198,13 @@ class LSMOSTOLineScan(DownloadableDataset):
"""
+
...
class LSMOSTOLineScanLowLoss(DownloadableDataset):
"""
LSMOSTOLineScanLowLoss
-
+
The low-loss half of the DualEELS pair for LSMOSTOLineScan - a La(0.7)Sr(0.3)MnO3 on SrTiO3 thin film line scan. 10 probe positions at 3.152 nm with 512 energy channels covering -50 to 461 eV at 1 eV dispersion, so it contains the zero-loss peak and the plasmon region. Use it with the core-loss spectrum for relative thickness mapping and Fourier-ratio deconvolution. Acquired on a JEOL ARM200cF with a Gatan Quantum ER at 200 kV; the low-loss dwell time is 9.44e-05 s against 0.4999 s for the core loss.
License: Unspecified
@@ -219,13 +214,14 @@ class LSMOSTOLineScanLowLoss(DownloadableDataset):
"""
+
...
-class MgONanoCrystals(DownloadableDataset):
+class AlNanocrystals(DownloadableDataset):
"""
- MgONanoCrystals
-
- A 4D STEM dataset of various MgO nanocrystals
+ AlNanocrystals
+
+ A 4D STEM dataset of Al nanocrystals on a carbon support.
License: CC-BY-4.0
@@ -234,30 +230,32 @@ class MgONanoCrystals(DownloadableDataset):
"""
+
...
-class NiEBSDLarge(DownloadableDataset):
+class HREBSDStrainPatterns(DownloadableDataset):
"""
- NiEBSDLarge
-
- 4125 EBSD patterns in a (55, 75) navigation shape of (60, 60) pixels from nickel, acquired on a NORDIF UF-1100 detector
+ HREBSDStrainPatterns
+
+ High-resolution EBSD patterns collected on a Direct Electron DE-Meridian, centred on a deformed region suitable for cross-correlation strain analysis. 32 x 32 probe positions at a 25 nm step, each pattern the sum of 256 counted, dark- and gain-corrected frames, binned 2 x 2 from 2048 x 2048 to 1024 x 1024 pixels. The pattern-plane geometry (detector distance, pattern centre, sample tilt) and the accelerating voltage were not recorded with the raw data, so the pattern axes are in detector pixels.
+
+ DOI: 10.5281/zenodo.21632101
License: CC-BY-4.0
You can download this dataset here:
- https://raw.githubusercontent.com/pyxem/kikuchipy-data/bcab8f7a4ffdb86a97f14e2327a4813d3156a85e/nickel_ebsd_large/
+ https://zenodo.org/records/21632101/files
"""
+
...
-class PdCuSiCrystallization(DownloadableDataset):
+class ZrNbPrecipitate(DownloadableDataset):
"""
- PdCuSiCrystallization
-
- A time resolved 4D-STEM series following the crystallization of a PdCuSi metallic glass, acquired on a Direct Electron CeleritasXS at 40000 fps with a 25 microsecond dwell time. 400 sequential scans of 47 x 39 probe positions (a 23.5 x 19.5 nm region cropped from a 256 x 256 scan) of 128 x 128 pixel diffraction patterns. Both real and reciprocal space are calibrated - 0.5 nm per scan step and 0.11 1/nm per detector pixel, centred on the direct beam. Successive scans are 1.6384 s apart and span 655.36 s of elapsed time, although the time axis is stored with a nm unit label. 24 GB of uint16 data uncompressed, chunked one time step at a time - the dataset used in the pyxem large data and lazy processing demo.
+ ZrNbPrecipitate
- DOI: 10.5281/zenodo.15490547
+ A 4D STEM dataset of ZrNb precipitate in ZrNb alloy.
License: CC-BY-4.0
@@ -266,13 +264,14 @@ class PdCuSiCrystallization(DownloadableDataset):
"""
+
...
-class PdNiPGlass(DownloadableDataset):
+class MgONanoCrystals(DownloadableDataset):
"""
- PdNiPGlass
-
- A 4D STEM dataset of PdNiP metallic glass thin film.
+ MgONanoCrystals
+
+ A 4D STEM dataset of various MgO nanocrystals
License: CC-BY-4.0
@@ -281,13 +280,16 @@ class PdNiPGlass(DownloadableDataset):
"""
+
...
-class SPEDAg(DownloadableDataset):
+class PdCuSiCrystallization(DownloadableDataset):
"""
- SPEDAg
-
- A 4D STEM dataset of polycrystalline Ag including twins and grain boundaries.
+ PdCuSiCrystallization
+
+ A time resolved 4D-STEM series following the crystallization of a PdCuSi metallic glass, acquired on a Direct Electron CeleritasXS at 40000 fps with a 25 microsecond dwell time. 400 sequential scans of 47 x 39 probe positions (a 23.5 x 19.5 nm region cropped from a 256 x 256 scan) of 128 x 128 pixel diffraction patterns. Both real and reciprocal space are calibrated - 0.5 nm per scan step and 0.11 1/nm per detector pixel, centred on the direct beam. Successive scans are 1.6384 s apart and span 655.36 s of elapsed time, although the time axis is stored with a nm unit label. 24 GB of uint16 data uncompressed, chunked one time step at a time - the dataset used in the pyxem large data and lazy processing demo.
+
+ DOI: 10.5281/zenodo.15490547
License: CC-BY-4.0
@@ -296,21 +298,60 @@ class SPEDAg(DownloadableDataset):
"""
+
...
-class ZrNbPrecipitate(DownloadableDataset):
+class CuZnHAADF(DownloadableDataset):
"""
- ZrNbPrecipitate
-
- A 4D STEM dataset of ZrNb precipitate in ZrNb alloy.
+ CuZnHAADF
- License: CC-BY-4.0
+ The HAADF survey image acquired simultaneously with the CuZnEELSMapping spectrum image - copper and zinc oxide on carbon nanotubes. 40 x 50 pixels at 0.9214 nm, uint16, on a JEOL ARM200F at 200 kV and 600000x. Pairs with CuZnEELSMapping for correlating elemental maps against the survey. Note - the Sample.description field in the file reads "Ta2O5 25% TiO2 CSIRO 400C", which is stale metadata carried over from an unrelated acquisition.
+
+ License: Unspecified
You can download this dataset here:
- https://zenodo.org/records/15490547/files
+ https://raw.githubusercontent.com/hyperspy/exspy-demos/927d1f21b3b8aba4e2e622c2e621d3d9d5542d1c/EELS/datasets
+
+
+ """
+ ...
+class CuZnEELSMapping(DownloadableDataset):
"""
+ CuZnEELSMapping
+
+ An EELS spectrum image of copper and zinc oxide deposited on carbon nanotubes, used by the eXSpy elemental mapping tutorial. 40 x 50 probe positions at 0.9214 nm with 162 energy channels covering 700-1988 eV at 8 eV dispersion, which resolves the Cu-L2,3 (~931 eV) and Zn-L2,3 (~1020 eV) edges. The Zn:Cu ratio is 3:1 and roughly 80 wt% of the sample is carbon, so the edges sit on a large plasmon background - a good test case for model-based background removal and overlapping-edge quantification. The simultaneously acquired survey image is available as CuZnHAADF. Note - the Sample.description field in the file reads "Ta2O5 25% TiO2 CSIRO 400C", which is stale metadata carried over from an unrelated acquisition.
+
+ License: Unspecified
+
+ You can download this dataset here:
+ https://raw.githubusercontent.com/hyperspy/exspy-demos/927d1f21b3b8aba4e2e622c2e621d3d9d5542d1c/EELS/datasets
+
+
+ """
+
...
-__all__ = __all__ + ['AlNanocrystals', 'AmorphousFilm4nm4DSTEM', 'ApoferritinApollo15eps', 'BilayerWS2', 'CuZnEELSMapping', 'CuZnHAADF', 'FeAlStripes', 'HREBSDStrainPatterns', 'InSituElectrochemGrowth', 'LayeredCuNb4DSTEM', 'LSMOLineScan', 'LSMOLineScanLowLoss', 'LSMOSTOLineScan', 'LSMOSTOLineScanLowLoss', 'MgONanoCrystals', 'NiEBSDLarge', 'PdCuSiCrystallization', 'PdNiPGlass', 'SPEDAg', 'ZrNbPrecipitate']
\ No newline at end of file
+__all__ = [
+ "LSMOLineScan",
+ "PdNiPGlass",
+ "FeAlStripes",
+ "InSituElectrochemGrowth",
+ "ApoferritinApollo15eps",
+ "AmorphousFilm4nm4DSTEM",
+ "SPEDAg",
+ "LayeredCuNb4DSTEM",
+ "BilayerWS2",
+ "NiEBSDLarge",
+ "LSMOLineScanLowLoss",
+ "LSMOSTOLineScan",
+ "LSMOSTOLineScanLowLoss",
+ "AlNanocrystals",
+ "HREBSDStrainPatterns",
+ "ZrNbPrecipitate",
+ "MgONanoCrystals",
+ "PdCuSiCrystallization",
+ "CuZnHAADF",
+ "CuZnEELSMapping",
+]
diff --git a/em_database/downloadable_dataset.py b/em_database/downloadable_dataset.py
index ad7cfbe..7e285ea 100644
--- a/em_database/downloadable_dataset.py
+++ b/em_database/downloadable_dataset.py
@@ -1,17 +1,33 @@
+import os
import threading
from concurrent.futures import Future, ThreadPoolExecutor
from pathlib import Path
-from typing import Optional, Union
+from typing import Any, ClassVar, Protocol
import pooch
-import os
+
+
+class Progress(Protocol):
+ """What pooch drives while streaming a file, and what the widgets provide."""
+
+ @property
+ def total(self) -> int: ...
+
+ @total.setter
+ def total(self, value: int) -> None: ...
+
+ def update(self, n: int) -> None: ...
+
+ def reset(self) -> None: ...
+
+ def close(self) -> None: ...
# A single, lazily-created thread pool shared by every dataset. Background
# downloads run here so that a notebook cell returns immediately instead of
# blocking on the network. It is created on first use so that simply importing
# the package costs nothing.
-_executor: Optional[ThreadPoolExecutor] = None
+_executor: ThreadPoolExecutor | None = None
_executor_lock = threading.Lock()
@@ -35,9 +51,26 @@ def _get_executor() -> ThreadPoolExecutor:
# the download path — but a Path subclass does.
_ConcretePath = type(Path())
+# Downloads in flight, keyed by the destination path. Keeping this off the
+# instance is what makes a derived path behave: pathlib builds a brand new
+# object for ``handle.parent / handle.name``, and an instance attribute would
+# not survive that, so the copy would report itself finished and never wait.
+_PENDING: dict[str, "Future[Path]"] = {}
+_PENDING_LOCK = threading.Lock()
+
+
+def _pending_key(path: object) -> str:
+ return os.path.normcase(os.path.abspath(str(path)))
+
-class DownloadFuture(_ConcretePath):
- """The path to a dataset that is downloading on a background thread.
+def _release_pending(key: str, future: "Future[Path]") -> None:
+ with _PENDING_LOCK:
+ if _PENDING.get(key) is future:
+ del _PENDING[key]
+
+
+class DatasetPath(_ConcretePath):
+ """The local path to a dataset, which may still be downloading.
It is a genuine :class:`pathlib.Path` pointing at the file's final location
(known before the download starts), so it can be passed anywhere a path is
@@ -46,69 +79,83 @@ class DownloadFuture(_ConcretePath):
every file reader ultimately goes through) it blocks until the download has
finished, re-raising any error that occurred.
- For explicit control it also behaves like a future: ``done()`` checks status
- without blocking, ``result()``/``wait()`` block until the file is ready.
+ ``done`` reports status without blocking; ``result()``/``wait()`` block
+ until the file is ready. A path to a file that is already on disk is simply
+ one that is already done, so :meth:`DownloadableDataset.download` returns
+ this type whether or not it downloaded anything.
+
+ Any path pointing at the same file waits, however it was built. ``str()``
+ and ``Path()`` are the exceptions: they hand back a plain value with no
+ download attached, so ``hs.load(str(handle))`` will not block.
"""
- def _attach(self, future: "Future[str]") -> "DownloadFuture":
- self._future = future
+ @property
+ def _future(self) -> "Future[Path] | None":
+ return _PENDING.get(_pending_key(self))
+
+ def _attach(self, future: "Future[Path]") -> "DatasetPath":
+ key = _pending_key(self)
+ with _PENDING_LOCK:
+ _PENDING[key] = future
+ future.add_done_callback(lambda finished: _release_pending(key, finished))
return self
def __fspath__(self) -> str:
# is_file()/exists()/stat()/open() and every os.fspath() consumer route
# through here, so blocking here makes all of them wait for the bytes.
- future = getattr(self, "_future", None)
+ future = self._future
if future is not None:
future.result() # blocks; re-raises a failed download
return str(self)
- def result(self, timeout: Optional[float] = None) -> str:
+ def result(self, timeout: float | None = None) -> Path:
"""Block until the download finishes and return the file path."""
- future = getattr(self, "_future", None)
+ future = self._future
if future is not None:
future.result(timeout)
- return str(self)
+ return Path(str(self))
+ @property
def done(self) -> bool:
- """Return True if the download has finished (without blocking)."""
- future = getattr(self, "_future", None)
+ """Whether the download has finished. Never blocks."""
+ future = self._future
return future.done() if future is not None else True
- def wait(self, timeout: Optional[float] = None) -> "DownloadFuture":
+ def wait(self, timeout: float | None = None) -> "DatasetPath":
"""Block until the download finishes and return self (for chaining)."""
self.result(timeout)
return self
def __repr__(self) -> str: # never block just to display the object
- state = "done" if self.done() else "downloading"
- return f""
+ state = "done" if self.done else "downloading"
+ return f""
class DownloadableDataset:
+ """A downloadable dataset, described by the YAML entry in :attr:`_spec`.
+
+ The generated subclasses in :mod:`em_database.data` carry their entry as
+ ``_spec``; keyword arguments override it for a single instance.
+ """
- def __init__(self,
- source: str,
- file: str,
- checksum:str=None,
- license:str=None,
- quality:str=None,
- data_size:str=None,
- doi:str=None,
- description:str=None,
- detector:Optional[str]=None,
- detector_manufacturer:Optional[str]=None,
- **kwargs):
- self.source = source
- self.file = file
- self.checksum = checksum
- self.license = license
- self.quality = quality
- self.doi = doi
- self.data_size = data_size
- self.description = description
- self.metadata = kwargs
- self.detector_manufacturer = detector_manufacturer
- self.detector = detector
+ _spec: ClassVar[dict[str, Any]] = {}
+
+ def __init__(self, **overrides: Any):
+ spec = {**self._spec, **overrides}
+ try:
+ self.source = spec.pop("source")
+ self.file = spec.pop("file")
+ except KeyError as error:
+ raise TypeError(f"a dataset needs a {error} entry") from None
+ self.checksum = spec.pop("checksum", None)
+ self.license = spec.pop("license", None)
+ self.quality = spec.pop("quality", None)
+ self.doi = spec.pop("doi", None)
+ self.data_size = spec.pop("data_size", None)
+ self.description = spec.pop("description", None)
+ self.detector_manufacturer = spec.pop("detector_manufacturer", None)
+ self.detector = spec.pop("detector", None)
+ self.metadata = spec
def __repr__(self):
return f"<{self.__class__} url={self.source}/{self.file} bytes={self.data_size}>"
@@ -120,25 +167,29 @@ def _repr_mimebundle_(self, **kwargs):
"""
try:
from em_database.widget import card
+
widget = card(self)
except Exception:
return {"text/plain": repr(self)}
return widget._repr_mimebundle_(**kwargs)
@staticmethod
- def _resolve_destination(destination: str | None) -> str:
+ def _resolve_destination(destination: str | os.PathLike | None) -> Path:
"""Return the directory the dataset should live in."""
if destination is None:
from em_database import config
+
return config.data_dir()
- return destination
+ return Path(destination)
- def download(self,
- destination: str | None = None,
- progressbar:bool = True,
- chunk_size:int =4096,
- background:bool = True) -> Union[str, DownloadFuture]:
- """ Download the dataset to the specified destination if not already present.
+ def download(
+ self,
+ destination: str | os.PathLike | None = None,
+ progressbar: bool | Progress = True,
+ chunk_size: int = 4096,
+ background: bool = True,
+ ) -> DatasetPath:
+ """Download the dataset to the specified destination if not already present.
By default, this will download to the defined emdata.data_dir directory. You can set
a custom default download directory with emdata.data_dir = 'your/path/here' which will
@@ -149,7 +200,7 @@ def download(self,
Parameters
----------
- destination : str, optional
+ destination : str or Path, optional
The directory to download the dataset to. If None, uses the default emdata.data_dir
directory, by default None.
progressbar : bool, optional
@@ -160,49 +211,48 @@ def download(self,
background : bool, optional
If True (the default), the download runs on a background thread and this
returns immediately so a Jupyter cell stays responsive. The returned
- :class:`DownloadFuture` is a real path pointing at the file's final
+ :class:`DatasetPath` is a real path pointing at the file's final
location, so you can hand it straight to a loader
(``hs.load(dataset.download())``): it blocks only at the point the file
- is actually opened. Use ``.done()`` to poll and ``.result()`` to wait
- explicitly. If False, the download blocks and returns the file path as a
- plain string, exactly as before.
+ is actually opened. Use ``.done`` to poll and ``.result()`` to wait
+ explicitly. If False, the download blocks until the file is there.
Returns
-------
- str or DownloadFuture
- When ``background`` is False, the local path to the downloaded file as a
- string. When ``background`` is True (the default), a :class:`DownloadFuture`
- path handle for the same location that resolves once the download finishes.
+ DatasetPath
+ The local path to the file, as a :class:`pathlib.Path` subclass that also
+ reports download state. With ``background`` False it is already done.
"""
if not background:
- return self._retrieve(destination, progressbar, chunk_size)
+ return DatasetPath(self._retrieve(destination, progressbar, chunk_size))
# Resolve where the file will end up: an existing shared/user copy if it
# is already present, otherwise the user's download location.
if destination is not None:
- target = os.path.join(destination, self.file)
+ target = Path(destination) / self.file
else:
- target = self.filepath() or os.path.join(self._resolve_destination(None), self.file)
+ target = self.filepath() or self._resolve_destination(None) / self.file
# In Jupyter (with the widget installed) a background download pops a
# cancelable toast; the toast's monitor replaces the plain progress bar.
monitor = finish = None
if progressbar:
try:
from em_database.widget import _attach_toast
+
monitor, finish = _attach_toast(type(self).__name__)
except Exception:
monitor = finish = None
progress = monitor if monitor is not None else progressbar
- future = _get_executor().submit(
- self._retrieve, destination, progress, chunk_size
- )
+ future = _get_executor().submit(self._retrieve, destination, progress, chunk_size)
if finish is not None:
future.add_done_callback(finish)
- return DownloadFuture(target)._attach(future)
-
- def _retrieve(self,
- destination: str | None = None,
- progressbar: bool = True,
- chunk_size: int = 4096) -> str:
+ return DatasetPath(target)._attach(future)
+
+ def _retrieve(
+ self,
+ destination: str | os.PathLike | None = None,
+ progressbar: bool | Progress = True,
+ chunk_size: int = 4096,
+ ) -> Path:
"""Fetch the file and return its local path (blocking).
With no explicit destination, an existing system-wide/shared copy is used
@@ -224,45 +274,49 @@ def _retrieve(self,
destination = self._resolve_destination(destination)
# Instantiate an Http downloader with a custom user agent
headers = {"User-Agent": "em_database (https://github.com/CSSFrancis/em_data)"}
- downloader = pooch.HTTPDownloader(progressbar=progressbar,
- chunk_size=chunk_size,
- headers= headers)
+ downloader = pooch.HTTPDownloader(
+ progressbar=progressbar, # pyright: ignore[reportArgumentType]
+ chunk_size=chunk_size,
+ headers=headers,
+ )
filepath = pooch.retrieve(
- url=self.source +"/"+ self.file,
+ url=self.source + "/" + self.file,
known_hash=self.checksum,
fname=self.file,
path=destination,
- downloader=downloader
+ downloader=downloader, # pyright: ignore[reportArgumentType]
)
- return filepath
+ return Path(filepath)
- def _find_shared(self) -> str | None:
+ def _find_shared(self) -> Path | None:
"""Path to an existing copy in a shared/system data dir, or None."""
from em_database import config
+
for directory in config.shared_data_dirs():
- candidate = os.path.join(directory, self.file)
- if os.path.exists(candidate):
+ candidate = directory / self.file
+ if candidate.exists():
return candidate
return None
- def filepath(self) -> str:
- """ Return the local file path of the dataset if present.
+ def filepath(self) -> Path | None:
+ """Return the local file path of the dataset if present.
Looks in the shared/system data locations first, then the user's data
- directory. Returns None if the dataset is not downloaded anywhere. """
+ directory. Returns None if the dataset is not downloaded anywhere."""
from em_database import config
+
for directory in config.data_search_dirs():
- candidate = os.path.join(directory, self.file)
- if os.path.exists(candidate):
+ candidate = directory / self.file
+ if candidate.exists():
return candidate
return None
def delete(self, destination: str | None = None) -> bool:
- """ Delete the downloaded file if it is present.
+ """Delete the downloaded file if it is present.
Parameters
----------
- destination : str, optional
+ destination : str or Path, optional
The directory the dataset was downloaded to. If None, uses the
default emdata.data_dir directory, by default None.
@@ -271,8 +325,8 @@ def delete(self, destination: str | None = None) -> bool:
bool
True if a file was removed, False if there was nothing to delete.
"""
- path = os.path.join(self._resolve_destination(destination), self.file)
- if os.path.exists(path):
- os.remove(path)
+ path = self._resolve_destination(destination) / self.file
+ if path.exists():
+ path.unlink()
return True
return False
diff --git a/em_database/static/browser.css b/em_database/static/browser.css
index 3ba9959..8f1f27b 100644
--- a/em_database/static/browser.css
+++ b/em_database/static/browser.css
@@ -116,6 +116,7 @@
.emdb-glyph { flex: 0 0 auto; width: 12px; text-align: center; font-size: 10px; }
.emdb-glyph.on { color: var(--emdb-green); }
.emdb-glyph.off { color: var(--emdb-muted); }
+.emdb-glyph.shared { color: var(--emdb-blue); }
.emdb-name { font-weight: 600; white-space: nowrap; }
.emdb-meta { flex: 1; color: var(--emdb-subtext); font-size: 11px; text-align: right;
overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
@@ -137,6 +138,8 @@
.emdb-d-status { margin: 10px 0 12px; display: flex; align-items: center; gap: 10px; }
.emdb-d-badge { font-size: 11.5px; color: var(--emdb-subtext); }
.emdb-d-badge.on { color: var(--emdb-green); font-weight: 600; }
+.emdb-d-badge.shared { color: var(--emdb-blue); font-weight: 600; }
+.emdb-d-note { font-size: 11px; color: var(--emdb-muted); }
.emdb-delete {
cursor: pointer; font-family: var(--emdb-font); font-size: 11px; font-weight: 600;
color: var(--emdb-red); background: transparent;
diff --git a/em_database/static/browser.js b/em_database/static/browser.js
index b0be3d7..d92bb9e 100644
--- a/em_database/static/browser.js
+++ b/em_database/static/browser.js
@@ -204,8 +204,9 @@ function render({ model, el: root }) {
const isActive = active.has(item.name);
const row = el("div", "emdb-row" + (state.selected === item.name ? " selected" : ""));
const meta = [item.size, item.shape].filter(Boolean).join(" · ");
- row.appendChild(el("span", "emdb-glyph " + (item.downloaded ? "on" : "off"),
- item.downloaded ? "●" : "○"));
+ const glyph = el("span", "emdb-glyph " + glyphClass(item), item.downloaded ? "●" : "○");
+ if (item.location === "shared") glyph.title = "installed system-wide: " + item.path;
+ row.appendChild(glyph);
row.appendChild(el("span", "emdb-name", esc(item.name)));
row.appendChild(el("span", "emdb-meta", esc(meta)));
row.appendChild(drawAction(item, isActive));
@@ -215,6 +216,11 @@ function render({ model, el: root }) {
return row;
}
+ function glyphClass(item) {
+ if (!item.downloaded) return "off";
+ return item.location === "shared" ? "shared" : "on";
+ }
+
function drawAction(item, isActive) {
const wrap = el("span", "emdb-actions");
if (item.downloaded) {
@@ -251,7 +257,12 @@ function render({ model, el: root }) {
// status / action line
const statusRow = el("div", "emdb-d-status");
- if (item.downloaded) {
+ if (item.downloaded && item.location === "shared") {
+ const badge = el("span", "emdb-d-badge shared", "● shared");
+ badge.title = "installed system-wide: " + item.path;
+ statusRow.appendChild(badge);
+ statusRow.appendChild(el("span", "emdb-d-note", "installed for every user, not yours to delete"));
+ } else if (item.downloaded) {
statusRow.appendChild(el("span", "emdb-d-badge on", "● downloaded"));
const del = el("button", "emdb-delete", "Delete");
del.title = "Remove the downloaded file from disk";
diff --git a/em_database/static/card.js b/em_database/static/card.js
index d54aefa..cf590e7 100644
--- a/em_database/static/card.js
+++ b/em_database/static/card.js
@@ -124,6 +124,11 @@ function render({ model, el: root }) {
const status = el("div", "emdb-d-status");
if (downloading) {
status.appendChild(el("span", "emdb-d-badge", "downloading…"));
+ } else if (it.downloaded && it.location === "shared") {
+ const badge = el("span", "emdb-d-badge shared", "● shared");
+ badge.title = "installed system-wide: " + it.path;
+ status.appendChild(badge);
+ status.appendChild(el("span", "emdb-d-note", "installed for every user, not yours to delete"));
} else if (it.downloaded) {
status.appendChild(el("span", "emdb-d-badge on", "● downloaded"));
const del = el("button", "emdb-delete", "Delete");
diff --git a/em_database/tests/__init__.py b/em_database/tests/__init__.py
index bdab649..e69de29 100644
--- a/em_database/tests/__init__.py
+++ b/em_database/tests/__init__.py
@@ -1,7 +0,0 @@
-import pytest
-
-from em_database import get_data_dir, set_data_dir, reset_data_dir
-
-def test_download_directory():
- reset_data_dir()
- print(get_data_dir())
diff --git a/em_database/tests/conftest.py b/em_database/tests/conftest.py
index 68e55c5..a13dde5 100644
--- a/em_database/tests/conftest.py
+++ b/em_database/tests/conftest.py
@@ -4,6 +4,7 @@
the legacy ``EM_DATABASE_DATA_DIR`` env var cleared, so the developer's real
settings never affect a test and a test never writes to the real config.
"""
+
import pytest
diff --git a/em_database/tests/test_directory.py b/em_database/tests/test_directory.py
index d9ffbdc..1684ead 100644
--- a/em_database/tests/test_directory.py
+++ b/em_database/tests/test_directory.py
@@ -4,7 +4,7 @@
dataset in the index rather than a large one.
"""
-import os
+from pathlib import Path
import pytest
@@ -12,7 +12,7 @@
from em_database import data
from em_database.tests.test_load_data import TINY_DATASET
-DEFAULT_DIR = os.path.join(os.path.expanduser("~"), "em_database")
+DEFAULT_DIR = Path.home() / "em_database"
@pytest.fixture(autouse=True)
@@ -26,9 +26,14 @@ def test_get_data_dir():
assert em_database.get_data_dir() == DEFAULT_DIR
+def test_reset_data_dir_returns_to_the_default():
+ em_database.reset_data_dir()
+ assert em_database.get_data_dir() == DEFAULT_DIR
+
+
def test_set_data_dir(tmp_path):
em_database.set_data_dir(str(tmp_path))
- assert em_database.get_data_dir() == str(tmp_path)
+ assert em_database.get_data_dir() == tmp_path
def test_reset_data_dir(tmp_path):
@@ -42,7 +47,7 @@ def test_saving_to_configured_dir(tmp_path):
em_database.set_data_dir(str(tmp_path))
dataset = getattr(data, TINY_DATASET)()
dest = dataset.download(progressbar=False, background=False)
- assert os.path.exists(os.path.join(str(tmp_path), dataset.file))
+ assert (tmp_path / dataset.file).exists()
# a second download must reuse the file rather than refetch it
assert dataset.download(progressbar=False, background=False) == dest
@@ -53,7 +58,7 @@ def test_saving_to_explicit_dir(tmp_path):
em_database.set_data_dir(str(tmp_path / "configured"))
dataset = getattr(data, TINY_DATASET)()
dest = dataset.download(destination=str(other), progressbar=False, background=False)
- assert "elsewhere" in dest
+ assert "elsewhere" in str(dest)
assert (other / dataset.file).exists()
@@ -63,4 +68,4 @@ def test_filepath_reports_missing_and_present(tmp_path):
dataset = getattr(data, TINY_DATASET)()
assert dataset.filepath() is None
dataset.download(progressbar=False, background=False)
- assert dataset.filepath() == os.path.join(str(tmp_path), dataset.file)
+ assert dataset.filepath() == tmp_path / dataset.file
diff --git a/em_database/tests/test_load_data.py b/em_database/tests/test_load_data.py
index 723c0fd..e1dc817 100644
--- a/em_database/tests/test_load_data.py
+++ b/em_database/tests/test_load_data.py
@@ -20,13 +20,21 @@
import em_database.data as data
from em_database.data import MgONanoCrystals, NiEBSDLarge
-from em_database.downloadable_dataset import DownloadFuture
+from em_database.downloadable_dataset import (
+ _PENDING,
+ DatasetPath,
+ DownloadableDataset,
+ _pending_key,
+)
try:
- from quantem.core.io.file_readers import read_4dstem
+ from quantem.core.io.file_readers import ( # pyright: ignore[reportMissingImports]
+ read_4dstem,
+ )
QUANTEM_AVAILABLE = True
except ImportError:
+ read_4dstem = None
QUANTEM_AVAILABLE = False
# The smallest file in the index (34 kB). Used wherever a test needs a real
@@ -70,8 +78,7 @@ def test_metadata_is_complete(name):
assert dataset.file, f"{name} has no file"
assert dataset.description, f"{name} has no description"
assert dataset.checksum and dataset.checksum.startswith("md5:"), (
- f"{name} has no md5 checksum, so a corrupt or truncated download "
- f"would go unnoticed"
+ f"{name} has no md5 checksum, so a corrupt or truncated download would go unnoticed"
)
@@ -80,8 +87,10 @@ def test_download_verifies_checksum(tmp_path):
dataset = getattr(data, TINY_DATASET)()
path = dataset.download(destination=tmp_path, progressbar=False, background=False)
assert (tmp_path / dataset.file).exists()
- assert isinstance(path, str)
- assert path == str(tmp_path / dataset.file)
+ assert isinstance(path, DatasetPath)
+ assert isinstance(path, Path)
+ assert path.done is True # nothing pending, so the handle needs no waiting
+ assert path == tmp_path / dataset.file
def test_download_default_returns_path_handle(tmp_path):
@@ -89,13 +98,13 @@ def test_download_default_returns_path_handle(tmp_path):
that is a real ``Path`` and resolves to the downloaded file."""
dataset = getattr(data, TINY_DATASET)()
handle = dataset.download(destination=tmp_path, progressbar=False)
- assert isinstance(handle, DownloadFuture)
+ assert isinstance(handle, DatasetPath)
assert isinstance(handle, Path)
# Using it as a path blocks until the bytes are there, then behaves normally.
assert os.fspath(handle) == str(tmp_path / dataset.file)
assert handle.is_file()
assert (tmp_path / dataset.file).exists()
- assert handle.done()
+ assert handle.done
def test_download_handle_is_nonblocking_then_blocks_on_use(tmp_path, monkeypatch):
@@ -113,12 +122,84 @@ def slow_retrieve(destination=None, progressbar=True, chunk_size=4096):
monkeypatch.setattr(dataset, "_retrieve", slow_retrieve)
handle = dataset.download(destination=tmp_path, progressbar=False)
- assert started.wait(2) # the worker thread really started
- assert handle.done() is False # returned without waiting for it
+ assert started.wait(2) # the worker thread really started
+ assert handle.done is False # returned without waiting for it
assert not (tmp_path / dataset.file).exists()
# Consuming the path blocks until the worker finishes, then resolves.
assert Path(os.fspath(handle)).read_bytes() == b"payload"
- assert handle.done() is True
+ assert handle.done is True
+
+
+def test_download_handle_derived_paths_also_wait(tmp_path, monkeypatch):
+ """A path rebuilt from the handle names the same file, so it must wait too."""
+ dataset = getattr(data, TINY_DATASET)()
+ started = threading.Event()
+
+ def slow_retrieve(destination=None, progressbar=True, chunk_size=4096):
+ started.set()
+ time.sleep(0.4)
+ target = tmp_path / dataset.file
+ target.write_bytes(b"payload")
+ return str(target)
+
+ monkeypatch.setattr(dataset, "_retrieve", slow_retrieve)
+ handle = dataset.download(destination=tmp_path, progressbar=False)
+ assert started.wait(2)
+
+ derived = handle.parent / handle.name
+ assert derived is not handle
+ assert derived.done is False
+ assert Path(os.fspath(derived)).read_bytes() == b"payload"
+
+
+def test_a_path_that_is_not_downloading_never_waits(tmp_path, monkeypatch):
+ """Only the file being fetched is pending - its directory is not."""
+ dataset = getattr(data, TINY_DATASET)()
+
+ def slow_retrieve(destination=None, progressbar=True, chunk_size=4096):
+ time.sleep(0.3)
+ target = tmp_path / dataset.file
+ target.write_bytes(b"payload")
+ return str(target)
+
+ monkeypatch.setattr(dataset, "_retrieve", slow_retrieve)
+ handle = dataset.download(destination=tmp_path, progressbar=False)
+ assert handle.parent.done is True
+ handle.wait()
+
+
+def test_finished_downloads_leave_no_pending_entry(tmp_path):
+ dataset = getattr(data, TINY_DATASET)()
+ handle = dataset.download(destination=tmp_path, progressbar=False)
+ handle.wait()
+ key = _pending_key(handle)
+ # the done-callback that clears the entry runs just after result() returns
+ for _ in range(200):
+ if key not in _PENDING:
+ break
+ time.sleep(0.01)
+ assert key not in _PENDING
+
+
+def test_generated_class_can_be_subclassed():
+ base = getattr(data, TINY_DATASET)
+
+ class Subclass(base):
+ pass
+
+ assert Subclass().file == base().file
+
+
+def test_keyword_overrides_leave_the_class_spec_alone():
+ base = getattr(data, TINY_DATASET)
+ overridden = base(checksum="md5:" + "0" * 32)
+ assert overridden.checksum == "md5:" + "0" * 32
+ assert base().checksum != overridden.checksum
+
+
+def test_a_dataset_without_a_source_is_an_error():
+ with pytest.raises(TypeError):
+ DownloadableDataset()
def test_download_handle_propagates_errors(tmp_path):
@@ -165,6 +246,7 @@ def test_download_mgo_nanocrystals(tmp_path):
@pytest.mark.slow
@pytest.mark.skipif(not QUANTEM_AVAILABLE, reason="quantem is not installed")
def test_quantem_loading(tmp_path):
+ assert read_4dstem is not None
dataset = MgONanoCrystals()
file_path = dataset.download(destination=tmp_path, progressbar=False, background=False)
read_4dstem(file_path)
diff --git a/em_database/tests/test_settings.py b/em_database/tests/test_settings.py
index 36b1806..be1e22f 100644
--- a/em_database/tests/test_settings.py
+++ b/em_database/tests/test_settings.py
@@ -3,34 +3,37 @@
The autouse fixture in conftest.py isolates each test to an empty settings file
and clears the legacy env var, so these exercise the mechanism in isolation.
"""
+
+from pathlib import Path
+
import em_database
from em_database import config
def test_defaults_when_nothing_configured():
assert em_database.get_data_dir() == config._default_data_dir()
- assert em_database.settings["data_dir"] == config._default_data_dir()
+ assert em_database.settings["data_dir"] == str(config._default_data_dir())
def test_live_object_is_immediate_like_rcparams(tmp_path):
em_database.settings["data_dir"] = str(tmp_path / "live")
- assert em_database.get_data_dir() == str(tmp_path / "live") # no save needed
- assert not config.config_path().exists() # not persisted
+ assert em_database.get_data_dir() == tmp_path / "live" # no save needed
+ assert not config.config_path().exists() # not persisted
def test_set_data_dir_persists_across_sessions(tmp_path):
target = str(tmp_path / "data")
- em_database.set_data_dir(target) # persist=True by default
- assert em_database.get_data_dir() == target
+ em_database.set_data_dir(target) # persist=True by default
+ assert em_database.get_data_dir() == Path(target)
assert config._read_file()["data_dir"] == target
- config.settings.reload() # a fresh "session"
- assert em_database.get_data_dir() == target
+ config.settings.reload() # a fresh "session"
+ assert em_database.get_data_dir() == Path(target)
def test_set_data_dir_session_only_is_not_persisted(tmp_path):
target = str(tmp_path / "x")
em_database.set_data_dir(target, persist=False)
- assert em_database.get_data_dir() == target
+ assert em_database.get_data_dir() == Path(target)
assert not config.config_path().exists()
config.settings.reload()
assert em_database.get_data_dir() == config._default_data_dir() # forgotten
@@ -54,13 +57,13 @@ def test_generic_setting_roundtrips():
def test_saving_other_settings_keeps_the_default_dynamic():
"""Persisting an unrelated setting must NOT freeze the default data_dir into
the file - otherwise the default would stop being obeyed."""
- em_database.set_setting("quality", "high") # triggers a save()
+ em_database.set_setting("quality", "high") # triggers a save()
stored = config._read_file()
- assert stored == {"quality": "high"} # data_dir default not written
+ assert stored == {"quality": "high"} # data_dir default not written
assert em_database.get_data_dir() == config._default_data_dir()
def test_legacy_env_var_still_seeds(tmp_path, monkeypatch):
monkeypatch.setenv("EM_DATABASE_DATA_DIR", str(tmp_path / "fromenv"))
config.settings.reload() # a fresh "import" with the env var set
- assert em_database.get_data_dir() == str(tmp_path / "fromenv")
+ assert em_database.get_data_dir() == tmp_path / "fromenv"
diff --git a/em_database/tests/test_shared_data.py b/em_database/tests/test_shared_data.py
index 7beaf71..9e15660 100644
--- a/em_database/tests/test_shared_data.py
+++ b/em_database/tests/test_shared_data.py
@@ -4,15 +4,19 @@
env vars, so these exercise the resolution logic in isolation (no network - the
"downloads" here find a pre-placed file).
"""
+
import os
import em_database
from em_database import catalogue, config
+from em_database.downloadable_dataset import DownloadableDataset
from em_database.tests.test_load_data import TINY_DATASET
-def _dataset():
- return catalogue.resolve(TINY_DATASET)
+def _dataset() -> DownloadableDataset:
+ ds = catalogue.resolve(TINY_DATASET)
+ assert ds is not None
+ return ds
def test_settings_live_in_dot_em_database(monkeypatch):
@@ -30,11 +34,11 @@ def test_shared_dir_is_searched_before_the_user_dir(tmp_path, monkeypatch):
em_database.set_data_dir(str(user), persist=False)
ds = _dataset()
- assert ds.filepath() is None # nowhere yet
+ assert ds.filepath() is None # nowhere yet
(user / ds.file).write_bytes(b"user")
- assert ds.filepath() == str(user / ds.file) # found in the user dir
+ assert ds.filepath() == user / ds.file # found in the user dir
(shared / ds.file).write_bytes(b"shared")
- assert ds.filepath() == str(shared / ds.file) # shared/system wins
+ assert ds.filepath() == shared / ds.file # shared/system wins
def test_download_uses_a_shared_copy_without_refetching(tmp_path, monkeypatch):
@@ -45,21 +49,21 @@ def test_download_uses_a_shared_copy_without_refetching(tmp_path, monkeypatch):
em_database.set_data_dir(str(user), persist=False)
ds = _dataset()
- (shared / ds.file).write_bytes(b"payload") # pre-installed system-wide
+ (shared / ds.file).write_bytes(b"payload") # pre-installed system-wide
path = ds.download(background=False)
- assert path == str(shared / ds.file) # used the shared copy
- assert not (user / ds.file).exists() # nothing downloaded to the user dir
+ assert path == shared / ds.file # used the shared copy
+ assert not (user / ds.file).exists() # nothing downloaded to the user dir
def test_search_order_is_shared_then_user(tmp_path, monkeypatch):
a, b = tmp_path / "a", tmp_path / "b"
monkeypatch.setenv("EM_DATABASE_SHARED_DIR", os.pathsep.join([str(a), str(b)]))
em_database.set_data_dir(str(tmp_path / "user"), persist=False)
- assert config.data_search_dirs() == [str(a), str(b), str(tmp_path / "user")]
+ assert config.data_search_dirs() == [a, b, tmp_path / "user"]
def test_system_config_file_contributes_shared_dirs(tmp_path, monkeypatch):
system_file = tmp_path / "system.yaml"
system_file.write_text(f"data_dir: {tmp_path / 'sitewide'}\n", encoding="utf-8")
monkeypatch.setenv("EM_DATABASE_SYSTEM_CONFIG", str(system_file))
- assert str(tmp_path / "sitewide") in config.shared_data_dirs()
+ assert tmp_path / "sitewide" in config.shared_data_dirs()
diff --git a/em_database/tests/test_widget.py b/em_database/tests/test_widget.py
index 489d2c6..5c7de33 100644
--- a/em_database/tests/test_widget.py
+++ b/em_database/tests/test_widget.py
@@ -4,6 +4,7 @@
and read declared metadata). The widget tests need ``anywidget``; the one real
download is marked ``slow``.
"""
+
import threading
import pytest
@@ -18,6 +19,7 @@
# catalogue
# ---------------------------------------------------------------------------
+
def test_catalogue_groups_and_orders_by_technique():
cat = catalogue.catalogue()
assert cat["n_total"] > 0
@@ -33,18 +35,48 @@ def test_catalogue_groups_and_orders_by_technique():
def test_catalogue_entry_has_expected_fields():
ds = catalogue.resolve(TINY_DATASET)
+ assert ds is not None
row = catalogue.entry(TINY_DATASET, ds)
- for key in ("name", "technique", "size", "downloaded", "path", "description",
- "detector", "microscope", "voltage", "tags", "source", "file"):
+ for key in (
+ "name",
+ "technique",
+ "size",
+ "downloaded",
+ "path",
+ "description",
+ "detector",
+ "microscope",
+ "voltage",
+ "tags",
+ "source",
+ "file",
+ ):
assert key in row
assert row["name"] == TINY_DATASET
assert row["technique"] == "STEM"
assert isinstance(row["downloaded"], bool)
+def test_catalogue_entry_reports_where_the_file_came_from(tmp_path, monkeypatch):
+ shared, user = tmp_path / "shared", tmp_path / "user"
+ shared.mkdir()
+ user.mkdir()
+ monkeypatch.setenv("EM_DATABASE_SHARED_DIR", str(shared))
+ em_database.set_data_dir(str(user), persist=False)
+
+ ds = catalogue.resolve(TINY_DATASET)
+ assert ds is not None
+ assert catalogue.entry(TINY_DATASET, ds)["location"] is None
+ (user / ds.file).write_bytes(b"x")
+ assert catalogue.entry(TINY_DATASET, ds)["location"] == "user"
+ (shared / ds.file).write_bytes(b"x")
+ assert catalogue.entry(TINY_DATASET, ds)["location"] == "shared"
+
+
def test_catalogue_downloaded_flag_tracks_the_file(tmp_path):
em_database.set_data_dir(str(tmp_path), persist=False)
ds = catalogue.resolve(TINY_DATASET)
+ assert ds is not None
assert catalogue.entry(TINY_DATASET, ds)["downloaded"] is False
(tmp_path / ds.file).write_bytes(b"x") # pretend it is downloaded
assert catalogue.entry(TINY_DATASET, ds)["downloaded"] is True
@@ -54,6 +86,7 @@ def test_catalogue_downloaded_flag_tracks_the_file(tmp_path):
# widget
# ---------------------------------------------------------------------------
+
def _browser():
pytest.importorskip("anywidget")
return em_database.browse()
@@ -106,24 +139,32 @@ def test_command_update_routes_through_real_comm_handler(monkeypatch):
widget = _browser()
got = []
monkeypatch.setattr(widget, "_start_download", lambda name: got.append(name))
- msg = {"content": {"data": {"method": "update",
- "state": {"_command": {"action": "download", "name": "X", "nonce": 1}}}},
- "buffers": []}
+ msg = {
+ "content": {
+ "data": {
+ "method": "update",
+ "state": {"_command": {"action": "download", "name": "X", "nonce": 1}},
+ }
+ },
+ "buffers": [],
+ }
widget._handle_msg(msg) # the real ipywidgets handler
assert got == ["X"]
def test_search_blob_includes_authors_and_affiliation():
ds = catalogue.resolve("BilayerWS2")
+ assert ds is not None
row = catalogue.entry("BilayerWS2", ds)
- assert "nick hagopian" in row["search"] # author name
- assert "wisconsin" in row["search"] # author affiliation
- assert "4d-stem" in row["search"] # technique
+ assert "nick hagopian" in row["search"] # author name
+ assert "wisconsin" in row["search"] # author affiliation
+ assert "4d-stem" in row["search"] # technique
def test_delete_removes_downloaded_file(tmp_path):
em_database.set_data_dir(str(tmp_path), persist=False)
ds = catalogue.resolve(TINY_DATASET)
+ assert ds is not None
(tmp_path / ds.file).write_bytes(b"x")
assert ds.filepath() is not None
assert ds.delete() is True
@@ -160,6 +201,7 @@ def test_dataset_card_is_populated_and_routes(monkeypatch):
from em_database.widget import card
ds = catalogue.resolve(TINY_DATASET)
+ assert ds is not None
widget = card(ds)
assert widget.info["name"] == TINY_DATASET
assert widget.info["technique"] == "STEM"
@@ -172,8 +214,10 @@ def test_dataset_card_is_populated_and_routes(monkeypatch):
def test_dataset_display_is_a_widget_card():
pytest.importorskip("anywidget")
ds = catalogue.resolve(TINY_DATASET)
+ assert ds is not None
bundle = ds._repr_mimebundle_()
mimes = bundle[0] if isinstance(bundle, tuple) else bundle
+ assert mimes is not None
assert "application/vnd.jupyter.widget-view+json" in mimes
@@ -185,7 +229,9 @@ def _boom(_dataset):
monkeypatch.setattr(widget_mod, "card", _boom)
ds = catalogue.resolve(TINY_DATASET)
+ assert ds is not None
bundle = ds._repr_mimebundle_()
+ assert bundle is not None
assert "text/plain" in bundle
@@ -195,16 +241,16 @@ def test_settings_widget_edits_and_persists(tmp_path):
from em_database.widget import settings_widget
widget = settings_widget()
- assert widget.data_dir == em_database.get_data_dir()
+ assert widget.data_dir == str(em_database.get_data_dir())
widget._command = {"action": "save", "data_dir": str(tmp_path / "d"), "nonce": 1}
- assert em_database.get_data_dir() == str(tmp_path / "d")
- assert config._read_file()["data_dir"] == str(tmp_path / "d") # persisted
+ assert em_database.get_data_dir() == tmp_path / "d"
+ assert config._read_file()["data_dir"] == str(tmp_path / "d") # persisted
assert widget.data_dir == str(tmp_path / "d")
widget._command = {"action": "session", "data_dir": str(tmp_path / "s"), "nonce": 2}
- assert em_database.get_data_dir() == str(tmp_path / "s")
- assert config._read_file()["data_dir"] == str(tmp_path / "d") # NOT persisted
+ assert em_database.get_data_dir() == tmp_path / "s"
+ assert config._read_file()["data_dir"] == str(tmp_path / "d") # NOT persisted
widget._command = {"action": "reset", "nonce": 3}
assert em_database.get_data_dir() == config._default_data_dir()
@@ -214,6 +260,7 @@ def test_settings_display_is_a_widget():
pytest.importorskip("anywidget")
bundle = em_database.settings._repr_mimebundle_()
mimes = bundle[0] if isinstance(bundle, tuple) else bundle
+ assert mimes is not None
assert "application/vnd.jupyter.widget-view+json" in mimes
@@ -221,15 +268,17 @@ def test_notebook_detection_and_colab_enable_are_safe():
"""The frontend helpers must be no-ops off a notebook (e.g. under pytest),
so nothing breaks when em_database is imported in plain Python."""
import em_database.widget as widget_mod
+
assert widget_mod._in_notebook() is False
widget_mod._enable_colab_widgets() # must not raise when not on Colab
- widget_mod._prepare_frontend() # idempotent, safe
+ widget_mod._prepare_frontend() # idempotent, safe
def test_attach_toast_is_noop_outside_jupyter():
"""Outside a Jupyter kernel there is no toast, so a bare download is
unaffected."""
import em_database.widget as widget_mod
+
monitor, finish = widget_mod._attach_toast("Foo")
assert monitor is None and finish is None
@@ -246,17 +295,19 @@ def test_download_toasts_plumbing():
monitor, token = toasts.begin("Foo")
assert toasts.downloads[token] == {"label": "Foo", "done": 0, "total": 0}
- ok = Future(); ok.set_result("path")
- toasts.finish(token, ok) # success -> toast cleared
+ ok = Future()
+ ok.set_result("path")
+ toasts.finish(token, ok) # success -> toast cleared
assert token not in toasts.downloads
- monitor2, token2 = toasts.begin("Bar") # cancel sets the event
+ monitor2, token2 = toasts.begin("Bar") # cancel sets the event
toasts._command = {"action": "cancel", "token": token2, "nonce": 1}
assert monitor2._cancel.is_set()
- bad = Future(); bad.set_exception(RuntimeError("boom"))
+ bad = Future()
+ bad.set_exception(RuntimeError("boom"))
_, token3 = toasts.begin("Baz")
- toasts.finish(token3, bad) # failure -> error toast
+ toasts.finish(token3, bad) # failure -> error toast
assert toasts.downloads[token3]["error"] == "boom"
@@ -268,4 +319,5 @@ def test_widget_download_end_to_end(tmp_path):
assert future is not None
future.result(timeout=120) # block until the background download finishes
ds = catalogue.resolve(TINY_DATASET)
+ assert ds is not None
assert (tmp_path / ds.file).exists()
diff --git a/em_database/widget.py b/em_database/widget.py
index 6672cb3..d268dfa 100644
--- a/em_database/widget.py
+++ b/em_database/widget.py
@@ -10,6 +10,7 @@
is not installed. Importing this module never imports anywidget at module load,
so ``import em_database`` stays cheap and dependency-light.
"""
+
from __future__ import annotations
import itertools
@@ -41,7 +42,9 @@ def _quiet_pooch():
return
try:
import logging
+
import pooch
+
pooch.get_logger().setLevel(logging.WARNING)
_pooch_quieted = True
except Exception:
@@ -60,7 +63,8 @@ def _enable_colab_widgets():
if _colab_enabled:
return
try:
- from google.colab import output # importable only on Colab
+ from google.colab import output # pyright: ignore[reportMissingImports]
+
output.enable_custom_widget_manager()
except Exception:
pass
@@ -222,12 +226,8 @@ def _start_download(self, name):
# and it also covers the cached case where no bytes ever flow.
self._set_progress(token, name, 0, 0)
monitor = _WidgetProgress(self, token, name, cancel)
- future = _get_executor().submit(
- ds.download, progressbar=monitor, background=False
- )
- future.add_done_callback(
- lambda f, tk=token, nm=name: self._finish_download(tk, nm, f)
- )
+ future = _get_executor().submit(ds.download, progressbar=monitor, background=False)
+ future.add_done_callback(lambda f, tk=token, nm=name: self._finish_download(tk, nm, f))
return future
def _finish_download(self, token, name, future):
@@ -281,7 +281,7 @@ class DatasetCard(anywidget.AnyWidget):
_esm = _STATIC / "card.js"
_css = _STATIC / "browser.css"
- info = traitlets.Dict().tag(sync=True) # the catalogue entry() dict
+ info = traitlets.Dict().tag(sync=True) # the catalogue entry() dict
download = traitlets.Dict().tag(sync=True) # {label, done, total} | {} | {error}
_command = traitlets.Dict().tag(sync=True)
@@ -397,6 +397,7 @@ def browse(**kwargs):
# Global toasts: a bare ``ds.download()`` in Jupyter pops a cancelable toast
# ---------------------------------------------------------------------------
+
def _make_settings_class():
"""Build the ``SettingsWidget`` class, importing anywidget lazily."""
import anywidget
@@ -423,10 +424,10 @@ def __init__(self, **kwargs):
self.observe(self._on_command, names="_command")
def _refresh(self, status=""):
- self.data_dir = config.data_dir()
- self.default_dir = config._default_data_dir()
+ self.data_dir = str(config.data_dir())
+ self.default_dir = str(config._default_data_dir())
self.config_path = str(config.config_path())
- self.search_dirs = list(config.data_search_dirs())
+ self.search_dirs = [str(d) for d in config.data_search_dirs()]
self.status = status
def _on_command(self, change):
@@ -552,7 +553,8 @@ def _in_notebook():
"""True in a notebook frontend that can render widgets (Jupyter, Colab,
VS Code, ...), False in plain Python or a terminal IPython."""
try:
- from IPython import get_ipython
+ from IPython.core.getipython import get_ipython
+
ip = get_ipython()
if ip is None:
return False
@@ -583,6 +585,7 @@ def _get_toasts():
if _toasts is None:
_toasts = _toasts_class()
from IPython.display import display
+
display(_toasts)
except Exception:
return None
diff --git a/examples/changing_download_location.py b/examples/changing_download_location.py
index 17b539f..4313d94 100644
--- a/examples/changing_download_location.py
+++ b/examples/changing_download_location.py
@@ -24,7 +24,7 @@
# Change it and remember the choice across sessions (writes
# ``~/.em_database/settings.yaml``). ``set_data_dir`` persists by default; the
# equivalent low-level form is ``em_database.settings[...] = ...; save()``.
-em_database.set_data_dir("/big/disk/em_data") # set + persist
+em_database.set_data_dir("/big/disk/em_data") # set + persist
# em_database.settings["data_dir"] = "/big/disk/em_data" # the same thing
# em_database.settings.save()
print("Persisted directory:", em_database.get_data_dir())
diff --git a/examples/loading_hyperspy.py b/examples/loading_hyperspy.py
index b6e3b26..4f81a75 100644
--- a/examples/loading_hyperspy.py
+++ b/examples/loading_hyperspy.py
@@ -6,6 +6,7 @@
"""
import hyperspy.api as hs
+
from em_database.data import BilayerWS2
# Load a dataset using HyperSpy
diff --git a/examples/loading_quantem.py b/examples/loading_quantem.py
index 72067a6..8bb9baa 100644
--- a/examples/loading_quantem.py
+++ b/examples/loading_quantem.py
@@ -8,6 +8,7 @@
from quantem.core.io.file_readers import read_4dstem
from em_database.data import BilayerWS2
+
# Ensure the dataset is downloaded
dataset = BilayerWS2()
file_path = dataset.download()
@@ -16,4 +17,4 @@
print(data)
# %%
# Display the data using quantem's built-in visualization
-data.show()
\ No newline at end of file
+data.show()
diff --git a/pyproject.toml b/pyproject.toml
index 66cddb9..591f0ba 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,7 +40,28 @@ Issues = "https://github.com/CSSFrancis/em_data/issues"
widget = ["anywidget>=0.9"]
tests = ["pytest", "pytest-cov", "hyperspy", "quantem", "anywidget>=0.9"]
doc = ["sphinx", "sphinx-rtd-theme","sphinx-gallery", "quantem", "hyperspy","pydata_sphinx_theme","sphinx_design",]
+dev = [
+ "ruff",
+ "basedpyright",
+ "pre-commit",
+ "pytest",
+ "jsonschema",
+ "anywidget>=0.9",
+]
+
+
+[tool.ruff]
+line-length = 99
+extend-exclude = ["*.md"]
+
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F", "I"]
+[tool.basedpyright]
+typeCheckingMode = "standard"
+include = ["em_database"]
+exclude = ["**/__pycache__", "doc", "build"]
+pythonVersion = "3.12"
[tool.pytest.ini_options]
# `slow` tests pull down the large datasets. They are deselected by default so