Update material

Author: MalloryWittwer

.gitignore

@@ -30,4 +30,5 @@ wasm/
 __marimo__/
 layouts/
 *.pkl
-todo.txt
+todo.txt
+demo_digits_extraction.ipynb

inference.py

@@ -22,38 +22,49 @@
 
 
 def extract_digit_from_digit_roi(image: np.ndarray) -> np.ndarray:
-    image = img_as_ubyte(image)
+    """Assuming that a digit is drawn inside the ROI, we extract a crop around it and resize it to (50, 50)."""
+    image = img_as_ubyte(image)  # Convert to 8-bit
 
+    # Downscale by a factor 2
     resized = rescale(image, scale=0.5)
 
+    # Otsu threshold
     try:
         binary = resized > threshold_otsu(resized)
     except ValueError:
         return img_as_ubyte(np.zeros((50, 50)))
 
+    # Labelling
     labelled = label(binary)
 
+    # The white background is assumed to be the biggest object
     light_square = keep_n_biggest_objects(labelled, n=1)
 
+    # We assume that the digit is the biggest "hole" in that bright background
+    # => If the digit is connected to the border it won't be detected
     filled_light_square = ndi.binary_fill_holes(light_square)
-    
     binary_in_light_square = np.logical_and(filled_light_square, light_square == 0)
-    
     labelled_in_light_square = label(binary_in_light_square)
 
     n_objects = labelled_in_light_square.max()
     if n_objects == 0:
+        # Case where the ROI is empty
         digit_crop = resized
     else:
+        # Digit = biggest object
         digit = keep_n_biggest_objects(labelled_in_light_square, n=1)
+        # Extract bounding box
         df = pd.DataFrame(
             regionprops_table(digit, intensity_image=resized, properties=["bbox"])
         )
         digit_row = df.iloc[0]
+        # Crop around the digit
         digit_crop = resized[digit_row["bbox-0"] : digit_row["bbox-2"], digit_row["bbox-1"] : digit_row["bbox-3"]]
 
+    # Resize to a fixed size
     digit_crop = resize(digit_crop, output_shape=(50, 50))
 
+    # Convert to 8-bit
     digit_crop = img_as_ubyte(digit_crop)
 
     return digit_crop

intro_to_ml.py

@@ -147,7 +147,7 @@ def _():
     return (example_slider,)
 
 
-@app.cell
+@app.cell(hide_code=True)
 def _(example_slider):
     mo.md(f"""
     The selected value is: **{example_slider.value}**
@@ -283,6 +283,21 @@ def _(df, imread, np):
     return (images,)
 
 
+@app.cell(hide_code=True)
+def _(dataset_acquisition_fig):
+    mo.md(f"""
+    /// details | How was this image dataset created?
+
+    This dataset was created by imaging 3 by 4 grids of numbers written in a predefined order (the ground truth class labels were inferred from that order). The image processing steps used to extract regions of interest around the numbers are illustrated below:
+
+    {dataset_acquisition_fig}
+
+    The Python script used to implement these steps is available [here](https://github.com/EPFL-Center-for-Imaging/vision-workshop/blob/main/preprocessing.py).
+    ///
+    """)
+    return
+
+
 @app.cell(hide_code=True)
 def _(num_labels):
     mo.md(f"""
@@ -1435,6 +1450,14 @@ def _(x_train_projected):
     return (extent,)
 
 
+@app.cell(hide_code=True)
+def _(public_folder_path):
+    _img_src = public_folder_path / "assets" / "training_fig.svg"
+
+    dataset_acquisition_fig = mo.vstack([mo.image(src=_img_src, alt="training_fig", height=600, caption="How the dataset was created")], align="center")
+    return (dataset_acquisition_fig,)
+
+
 @app.cell(hide_code=True)
 def _(public_folder_path):
     _img_src = public_folder_path / "assets" / "inference.png"

preprocessing.py

@@ -1,3 +1,8 @@
+"""
+Script used to extract digit images and the corresponding ground truth labels from images of training grids captured by a camera device. 
+
+The ground truth labels are inferred from the position of the digits in the grid, which is predetermined.
+"""
 import os
 import sys
 from pathlib import Path
@@ -15,14 +20,15 @@
 
 
 def keep_n_biggest_objects(labelled: np.ndarray, n=1) -> np.ndarray:
+    """Remove all but the N biggest objects in a labelled array."""
     uniques, counts = np.unique(labelled, return_counts=1)
 
     # Ignore the background if it's there
     if uniques[0] == 0:
         uniques = uniques[1:]
         counts = counts[1:]
 
-    # Sort unique values by counts (descending), then extract the n unique values corresponding to the biggest objects
+    # Sort unique values by counts (descending), then extract the N unique values corresponding to the biggest objects
     biggest_labels = uniques[np.argsort(counts)[::-1][:n]]
 
     biggest_objects_filt = np.isin(labelled, biggest_labels)
@@ -33,7 +39,8 @@ def keep_n_biggest_objects(labelled: np.ndarray, n=1) -> np.ndarray:
     return biggest_objects_mask
 
 
-def detect_digit_squares(image: np.ndarray, grid_shape: tuple) -> pd.DataFrame:
+def detect_digit_rois_in_training_grid(image: np.ndarray, grid_shape: tuple) -> pd.DataFrame:
+    """Extract digits from the image of a training grid."""
     # Convert the image to gray
     gray = rgb2gray(image)
 
@@ -79,20 +86,15 @@ def detect_digit_squares(image: np.ndarray, grid_shape: tuple) -> pd.DataFrame:
     return df
 
 
-def extract_digit_from_digit_square(image: np.ndarray) -> np.ndarray:
+def extract_digit_from_digit_roi(image: np.ndarray) -> np.ndarray:
+    """Note: almost the same as in the inference script, but we print warnings."""
     image = img_as_ubyte(image)
-    
     resized = rescale(image, scale=0.5)
-    
     binary = resized > threshold_otsu(resized)
     labelled = label(binary)
-
     light_square = keep_n_biggest_objects(labelled, n=1)
-
     filled_light_square = ndi.binary_fill_holes(light_square)
-    
     binary_in_light_square = np.logical_and(filled_light_square, light_square == 0)
-    
     labelled_in_light_square = label(binary_in_light_square)
 
     n_objects = labelled_in_light_square.max()
@@ -108,49 +110,54 @@ def extract_digit_from_digit_square(image: np.ndarray) -> np.ndarray:
         digit_crop = resized[digit_row["bbox-0"] : digit_row["bbox-2"], digit_row["bbox-1"] : digit_row["bbox-3"]]
 
     digit_crop = resize(digit_crop, output_shape=(50, 50))
-    
     digit_crop = img_as_ubyte(digit_crop)
 
     return digit_crop
 
 
 if __name__ == "__main__":
+    # Run with, for example: `python preprocessing.py datasets/`
     _, root_img_folder = sys.argv
 
-    # We assume 3 rows and 4 columns, with labels in a fixed, given order:
+    # We assume that the images contain a (N, M) grid of digits in a predetermined order:
     grid_shape = (3, 4)
-    labels = np.array(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B"]).reshape(grid_shape)
+    class_labels = np.array(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B"]).reshape(grid_shape)
 
     root = Path(root_img_folder)
 
-    dst_root = root / "digits"
-    if not dst_root.exists():
-        os.mkdir(dst_root)
+    training_images_dir = root / "raw_training_images"
+    
+    dataset_dir = root / "dataset"
+    if not dataset_dir.exists():
+        os.mkdir(dataset_dir)
 
-    for lab in labels.ravel():
-        dst_dir = dst_root / lab
+    # Create a subdirectory for each class
+    for lab in class_labels.ravel():
+        dst_dir = dataset_dir / lab
         if not dst_dir.exists():
             os.mkdir(dst_dir)
 
-    for image_file in root.glob("*.jpg"):
+    # Iterate over the training images
+    for image_file in training_images_dir.glob("*.jpg"):
         print("---")
         print(image_file)
         print("---")
 
-        img = skimage.io.imread(image_file)
+        training_img = skimage.io.imread(image_file)
 
         # Extract `digit squares` crops, sorted by [centroid-0, centroid-1] so we can assume their class label.
-        df = detect_digit_squares(img, grid_shape)
+        df = detect_digit_rois_in_training_grid(training_img, grid_shape)
 
         # Add the `class` label
-        df["class"] = labels.ravel()
+        df["class"] = class_labels.ravel()
 
         for _, row in df.iterrows():
-            digit_square_img = row["intensity_image"]
+            digit_roi = row["intensity_image"]
+            digit_cls = row["class"]
 
-            digit = extract_digit_from_digit_square(digit_square_img)
+            digit_img = extract_digit_from_digit_roi(digit_roi)
 
-            # Save it in the corresponding class subfolder
-            dst_file = dst_root / row["class"] / f"{image_file.stem}.png"
+            # Save the digit image in the corresponding class subfolder
+            dst_file = dataset_dir / digit_cls / f"{image_file.stem}.png"
 
-            skimage.io.imsave(dst_file, digit)
+            skimage.io.imsave(dst_file, digit_img)