Cleanup of em_data in prep for more changes

.github/scripts/issue_to_yaml.py

@@ -1,8 +1,10 @@
-import sys
 import re
-import yaml
+import sys
 from pathlib import Path
 
+import yaml
+
+
 def parse_issue_body(text):
     # Simple regex-based parser for the fields
     fields = {
@@ -18,7 +20,7 @@ def parse_issue_body(text):
         "Accelerating Voltage": r"Accelerating Voltage\s*(.*)",
         "Dataset License": r"Dataset License\s*(.*)",
         "Technique": r"Technique\s*(.*)",
-        "Tags": r"Tags\s*(.*)"
+        "Tags": r"Tags\s*(.*)",
     }
     data = {}
     for key, pattern in fields.items():
@@ -29,11 +31,12 @@ def parse_issue_body(text):
             data[key] = ""
     return data
 
+
 def build_yaml(data):
     # Convert tags to list
     tags = [t.strip() for t in data["Tags"].split(",") if t.strip()]
     # Use author as dataset name (sanitize)
-    d_name = re.sub(r'\W+', '', data["Dataset Name"])
+    d_name = re.sub(r"\W+", "", data["Dataset Name"])
     yaml_data = {
         d_name: {
             "description": data["Description"],
@@ -46,13 +49,12 @@ def build_yaml(data):
             "license": data["Dataset License"],
             "technique": data["Technique"],
             "tags": tags,
-            "authors": {
-                data["Author"]: {}
-            }
+            "authors": {data["Author"]: {}},
         }
     }
     return yaml_data, dataset_name
 
+
 if __name__ == "__main__":
     issue_file = sys.argv[1]
     out_dir = Path(sys.argv[2])
@@ -63,4 +65,4 @@ def build_yaml(data):
     out_path = out_dir / f"{dataset_name}.yaml"
     with open(out_path, "w") as f:
         f.write("# $schema: ../json-schema.json\n")
-        yaml.dump(yaml_data, f, sort_keys=False)
+        yaml.dump(yaml_data, f, sort_keys=False)

.github/workflows/build.yml

@@ -9,6 +9,30 @@ env:
   MPLBACKEND: agg
 
 jobs:
+  lint:
+    name: lint and type check
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies and package
+        run: pip install -U -e .'[dev]'
+
+      - name: ruff check
+        run: ruff check --output-format=github .
+
+      - name: ruff format
+        run: ruff format --check --diff .
+
+      - name: basedpyright
+        run: basedpyright
+
   build-with-pip:
     name: ${{ matrix.os }}-py${{ matrix.python-version }}${{ matrix.LABEL }}
     runs-on: ${{ matrix.os }}

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.16.5
+    hooks:
+      - id: ruff-check
+        args: [--fix]
+      - id: ruff-format

doc/source/conf.py

@@ -5,14 +5,15 @@
 
 import sys
 from pathlib import Path
+
 # Import and run the build script
 from em_database._build_docs import (
-    parse_datasets,
-    generate_html_table,
+    generate_add_dataset_html,
+    generate_all_data_html,
     generate_browser_html,
+    generate_html_table,
     generate_landing_html,
-    generate_all_data_html,
-    generate_add_dataset_html,
+    parse_datasets,
 )
 
 # Add project root to path
@@ -21,10 +22,10 @@
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
-project = 'em_database'
-copyright = '2026, Carter Francis'
-author = 'Carter Francis'
-release = '0.4.0'
+project = "em_database"
+copyright = "2026, Carter Francis"
+author = "Carter Francis"
+release = "0.4.0"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
@@ -35,15 +36,14 @@
     "sphinx.ext.intersphinx",
     "sphinx.ext.napoleon",
     "sphinx_gallery.gen_gallery",
-    'sphinx_design',
+    "sphinx_design",
 ]
 
-templates_path = ['_templates']
+templates_path = ["_templates"]
 # intro.rst / datasets.rst are superseded by the generated landing + All Data
 # app pages; keep the files but leave them out of the build so they don't warn
 # about being orphaned.
-exclude_patterns = ['intro.rst', 'datasets.rst']
-
+exclude_patterns = ["intro.rst", "datasets.rst"]
 
 
 # -- Options for HTML output -------------------------------------------------
@@ -87,17 +87,18 @@
 # No left sidebar anywhere - keep every page a single, full-width column.
 html_sidebars = {"**": []}
 _unused_sidebars = {
-  "index": [],
-  "all_data": [],
-  "add_dataset": [],
-  "datasets": [],
+    "index": [],
+    "all_data": [],
+    "add_dataset": [],
+    "datasets": [],
 }
 
+
 def build_datasets_html(app, exception):
     """Generate datasets.html during Sphinx build"""
     if exception is not None:
         print(f"Build exception: {exception}")
-    datasets_path = Path(__file__).parent.parent.parent / 'em_database' / 'datasets'
+    datasets_path = Path(__file__).parent.parent.parent / "em_database" / "datasets"
     print(f"Looking for datasets at: {datasets_path.absolute()}")
     print(f"Path exists: {datasets_path.exists()}")
     if datasets_path.exists():
@@ -107,9 +108,9 @@ def build_datasets_html(app, exception):
     print(datasets)
     html_output = generate_html_table(datasets)
 
-    output_path = Path(app.outdir) / 'datasets_db.html'
+    output_path = Path(app.outdir) / "datasets_db.html"
     output_path.parent.mkdir(parents=True, exist_ok=True)
-    with output_path.open('w', encoding='utf-8') as f:
+    with output_path.open("w", encoding="utf-8") as f:
         f.write(html_output)
 
     # Generated, self-contained Catppuccin "app" pages. Each is written into the
@@ -118,20 +119,22 @@ def build_datasets_html(app, exception):
     # em_database.browse(). Each is guarded so a failure never kills the build.
     outdir = Path(app.outdir)
     pages = {
-        'index.html': generate_landing_html,
-        'all_data.html': generate_all_data_html,
-        'add_dataset.html': generate_add_dataset_html,
-        'datasets_browser.html': generate_browser_html,
+        "index.html": generate_landing_html,
+        "all_data.html": generate_all_data_html,
+        "add_dataset.html": generate_add_dataset_html,
+        "datasets_browser.html": generate_browser_html,
     }
     for filename, generator in pages.items():
         try:
-            (outdir / filename).write_text(generator(), encoding='utf-8')
+            (outdir / filename).write_text(generator(), encoding="utf-8")
             print(f"Wrote {filename}")
         except Exception as e:  # pragma: no cover - keep the build alive
             print(f"Could not build {filename}: {e}")
 
+
 def setup(app):
-    app.connect('build-finished', build_datasets_html)
+    app.connect("build-finished", build_datasets_html)
+
 
 # sphinx_gallery
 # --------------

em_database/__init__.py

@@ -1,9 +1,8 @@
 ### Example datasets ###
-import os
+from em_database import data
 from em_database.config import settings
 from em_database.downloadable_dataset import DownloadableDataset
-from em_database._create_stubs import build_docstring
-from em_database import data
+
 __all__ = []
 
 
@@ -17,8 +16,10 @@ def get_data_dir():
         Path to the example datasets directory.
     """
     from em_database import config
+
     return config.data_dir()
 
+
 def set_data_dir(path: str, persist: bool = True):
     """
     Set the directory where example datasets are stored.
@@ -35,6 +36,7 @@ def set_data_dir(path: str, persist: bool = True):
     if persist:
         settings.save()
 
+
 def reset_data_dir():
     """
     Reset the example datasets directory to the default location, clearing any
@@ -47,6 +49,7 @@ def get_setting(key: str, default=None):
     """Read a value from :data:`em_database.settings`."""
     return settings.get(key, default)
 
+
 def set_setting(key: str, value, persist: bool = True):
     """Set a value in :data:`em_database.settings`, persisting it by default."""
     settings[key] = value
@@ -66,18 +69,28 @@ def browse(**kwargs):
     ``display(em_database)`` renders the same browser.
     """
     from em_database.widget import browse as _browse
+
     return _browse(**kwargs)
 
 
-__all__ =  ['get_data_dir', 'set_data_dir', 'reset_data_dir',
-            'get_setting', 'set_setting', 'settings', 'browse', "data"]
+__all__ = [
+    "get_data_dir",
+    "set_data_dir",
+    "reset_data_dir",
+    "get_setting",
+    "set_setting",
+    "settings",
+    "browse",
+    "data",
+    "DownloadableDataset",
+]
 
 
 # Let ``display(em_database)`` render the browser. Reassigning the module's
 # __class__ to a ModuleType subclass is a supported pattern (see PEP 562) and is
 # what lets the package itself carry a rich Jupyter repr.
-import sys as _sys
-from types import ModuleType as _ModuleType
+import sys as _sys  # noqa: E402
+from types import ModuleType as _ModuleType  # noqa: E402
 
 
 class _EmDatabaseModule(_ModuleType):
@@ -92,7 +105,7 @@ def _repr_mimebundle_(self, include=None, exclude=None, **kwargs):
                     "em_database.browse()."
                 )
             }
-        return widget._repr_mimebundle_(include=include, exclude=exclude, **kwargs)
+        return widget._repr_mimebundle_(**kwargs)
 
 
-_sys.modules[__name__].__class__ = _EmDatabaseModule
+_sys.modules[__name__].__class__ = _EmDatabaseModule