diff --git a/examples/vision/image_classification_efficientnet_fine_tuning.py b/examples/vision/image_classification_efficientnet_fine_tuning.py
index 2413b0c0de..6014bb6612 100644
--- a/examples/vision/image_classification_efficientnet_fine_tuning.py
+++ b/examples/vision/image_classification_efficientnet_fine_tuning.py
@@ -2,7 +2,7 @@
Title: Image classification via fine-tuning with EfficientNet
Author: [Yixing Fu](https://github.com/yixingfu)
Date created: 2020/06/30
-Last modified: 2023/07/10
+Last modified: 2026/07/13
Description: Use EfficientNet with weights pre-trained on imagenet for Stanford Dogs classification.
Accelerator: GPU
"""
@@ -25,12 +25,6 @@
efficiency-oriented base model (B0) to surpass models at every scale, while avoiding
extensive grid-search of hyperparameters.
-A summary of the latest updates on the model is available at
-[here](https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet), where various
-augmentation schemes and semi-supervised learning approaches are applied to further
-improve the imagenet performance of the models. These extensions of the model can be used
-by updating weights without changing model architecture.
-
## B0 to B7 variants of EfficientNet
*(This section provides some details on "compound scaling", and can be skipped
@@ -57,7 +51,7 @@
are hand-picked and proven to produce good results, though they may be significantly
off from the compound scaling formula.
Therefore, the keras implementation (detailed below) only provide these 8 models, B0 to B7,
-instead of allowing arbitray choice of width / depth / resolution parameters.
+instead of allowing arbitrary choice of width / depth / resolution parameters.
## Keras implementation of EfficientNet
@@ -65,7 +59,7 @@
use EfficientNetB0 for classifying 1000 classes of images from ImageNet, run:
```python
-from tensorflow.keras.applications import EfficientNetB0
+from keras.applications import EfficientNetB0
model = EfficientNetB0(weights='imagenet')
```
@@ -123,10 +117,13 @@
## Setup and data loading
"""
+import tarfile
+import urllib.request
+from pathlib import Path
import numpy as np
-import tensorflow_datasets as tfds
-import tensorflow as tf # For tf.data
import matplotlib.pyplot as plt
+from PIL import Image
+import scipy.io
import keras
from keras import layers
from keras.applications import EfficientNetB0
@@ -139,40 +136,127 @@
"""
### Loading data
-Here we load data from [tensorflow_datasets](https://www.tensorflow.org/datasets)
-(hereafter TFDS).
-Stanford Dogs dataset is provided in
-TFDS as [stanford_dogs](https://www.tensorflow.org/datasets/catalog/stanford_dogs).
-It features 20,580 images that belong to 120 classes of dog breeds
+We download the Stanford Dogs dataset directly from Stanford's servers using the
+built-in `urllib` and `tarfile` modules.
+The dataset contains 20,580 images belonging to 120 classes of dog breeds
(12,000 for training and 8,580 for testing).
-By simply changing `dataset_name` below, you may also try this notebook for
-other datasets in TFDS such as
-[cifar10](https://www.tensorflow.org/datasets/catalog/cifar10),
-[cifar100](https://www.tensorflow.org/datasets/catalog/cifar100),
-[food101](https://www.tensorflow.org/datasets/catalog/food101),
-etc. When the images are much smaller than the size of EfficientNet input,
-we can simply upsample the input images. It has been shown in
-[Tan and Le, 2019](https://arxiv.org/abs/1905.11946) that transfer learning
-result is better for increased resolution even if input images remain small.
+The dataset is downloaded, extracted, and loaded into lists of NumPy arrays.
+Images have variable dimensions and will be resized to a uniform size in the data pipeline.
+
+**Note:** This direct download approach eliminates dependency conflicts that can
+occur with `tensorflow_datasets` in some environments (particularly Google Colab
+with protobuf version incompatibilities).
"""
-dataset_name = "stanford_dogs"
-(ds_train, ds_test), ds_info = tfds.load(
- dataset_name, split=["train", "test"], with_info=True, as_supervised=True
-)
-NUM_CLASSES = ds_info.features["label"].num_classes
+dataset_url = "http://vision.stanford.edu/aditya86/ImageNetDogs/images.tar"
+lists_url = "http://vision.stanford.edu/aditya86/ImageNetDogs/lists.tar"
+data_dir = Path("./stanford_dogs_data")
+data_dir.mkdir(exist_ok=True)
-"""
-When the dataset include images with various size, we need to resize them into a
-shared size. The Stanford Dogs dataset includes only images at least 200x200
-pixels in size. Here we resize the images to the input size needed for EfficientNet.
-"""
+def download_and_extract(url, extract_to):
+ filename = url.split("/")[-1]
+ filepath = data_dir / filename
+ if not filepath.exists():
+ print(f"Downloading {filename}...")
+ urllib.request.urlretrieve(url, filepath)
+ print(f"Extracting {filename}...")
+ with tarfile.open(filepath, "r") as tar:
+ tar.extractall(extract_to, filter="data")
+ return extract_to
+
+
+# Download dataset
+images_dir = download_and_extract(dataset_url, data_dir)
+lists_dir = download_and_extract(lists_url, data_dir)
+
-size = (IMG_SIZE, IMG_SIZE)
-ds_train = ds_train.map(lambda image, label: (tf.image.resize(image, size), label))
-ds_test = ds_test.map(lambda image, label: (tf.image.resize(image, size), label))
+# Parse train/test splits
+def load_file_list(filepath):
+ mat = scipy.io.loadmat(filepath)
+ return [item[0][0] for item in mat["file_list"]]
+
+
+train_files = load_file_list(data_dir / "train_list.mat")
+test_files = load_file_list(data_dir / "test_list.mat")
+
+# Build class name mapping
+all_files = train_files + test_files
+class_names = sorted(set([f.split("/")[0] for f in all_files]))
+class_to_idx = {name: idx for idx, name in enumerate(class_names)}
+NUM_CLASSES = len(class_names)
+
+print(
+ f"Found {NUM_CLASSES} classes, {len(train_files)} training images, {len(test_files)} test images"
+)
+
+
+# Prepare image paths and labels (lazy loading - no images loaded into memory yet)
+def prepare_paths_and_labels(file_list, base_dir):
+ image_paths, labels = [], []
+ for file_path in file_list:
+ class_name = file_path.split("/")[0]
+ img_path = base_dir / "Images" / file_path
+ if img_path.exists():
+ image_paths.append(str(img_path))
+ labels.append(class_to_idx[class_name])
+ return image_paths, np.array(labels)
+
+
+print("Preparing dataset paths...")
+train_image_paths, train_labels = prepare_paths_and_labels(train_files, data_dir)
+test_image_paths, test_labels = prepare_paths_and_labels(test_files, data_dir)
+print(f"Found {len(train_image_paths)} train and {len(test_image_paths)} test images")
+
+
+"""
+Each image can have a different shape, so we resize them to a shared input size
+for EfficientNet. In Keras 3, we do this in a backend-agnostic `PyDataset` pipeline
+using `keras.ops.image.resize`, which works across TensorFlow, JAX, and PyTorch backends,
+rather than the TensorFlow-specific `tf.data` mapping steps.
+
+Images are loaded lazily from disk in `__getitem__` to avoid loading all 20,580 images
+into memory at once, which would consume several gigabytes of RAM and cause OOM issues.
+"""
+
+
+class ResizeOnlyDataset(keras.utils.PyDataset):
+ def __init__(self, image_paths, labels, img_size, batch_size=1, **kwargs):
+ super().__init__(**kwargs)
+ self.image_paths = image_paths
+ self.labels = labels
+ self.img_size = img_size
+ self.batch_size = batch_size
+ self.indices = np.arange(len(labels))
+
+ def __len__(self):
+ return int(np.ceil(len(self.labels) / self.batch_size))
+
+ def __getitem__(self, idx):
+ batch_indices = self.indices[
+ idx * self.batch_size : (idx + 1) * self.batch_size
+ ]
+ batch_images = []
+ for i in batch_indices:
+ img = Image.open(self.image_paths[i]).convert("RGB")
+ img_array = np.array(img, dtype="float32")
+ img_resized = keras.ops.image.resize(
+ img_array, (self.img_size, self.img_size)
+ )
+ batch_images.append(np.array(img_resized))
+ batch_images = np.stack(batch_images)
+ batch_labels = self.labels[batch_indices]
+ if self.batch_size == 1:
+ return batch_images[0], batch_labels[0]
+ return batch_images, batch_labels
+
+
+# Preview stream with resized images for visualization below
+preview_train = ResizeOnlyDataset(
+ train_image_paths, train_labels, IMG_SIZE, batch_size=1
+)
+preview_test = ResizeOnlyDataset(test_image_paths, test_labels, IMG_SIZE, batch_size=1)
"""
### Visualizing the data
@@ -182,14 +266,15 @@
def format_label(label):
- string_label = label_info.int2str(label)
- return string_label.split("-")[1]
+ class_name = class_names[int(label)]
+ return class_name.split("-")[1] # Extract breed name from "n02085620-Chihuahua"
-label_info = ds_info.features["label"]
-for i, (image, label) in enumerate(ds_train.take(9)):
+for i, (image, label) in enumerate(preview_train):
+ if i >= 9:
+ break
ax = plt.subplot(3, 3, i + 1)
- plt.imshow(image.numpy().astype("uint8"))
+ plt.imshow(np.array(image).astype("uint8"))
plt.title("{}".format(format_label(label)))
plt.axis("off")
@@ -197,7 +282,8 @@ def format_label(label):
"""
### Data augmentation
-We can use the preprocessing layers APIs for image augmentation.
+We can use Keras preprocessing layers for image augmentation.
+These layers are backend-agnostic and can be used during both training and inference.
"""
img_augmentation_layers = [
@@ -215,64 +301,125 @@ def img_augmentation(images):
"""
-This `Sequential` model object can be used both as a part of
-the model we later build, and as a function to preprocess
-data before feeding into the model. Using them as function makes
-it easy to visualize the augmented images. Here we plot 9 examples
-of augmentation result of a given figure.
+The `img_augmentation` function can be used both as a part of the model
+we later build, and as a standalone function to preprocess data before feeding
+into the model. Using it as a function makes it easy to visualize the augmentation
+results. Here we plot 9 examples of augmentation applied to a single image.
"""
-for image, label in ds_train.take(1):
- for i in range(9):
- ax = plt.subplot(3, 3, i + 1)
- aug_img = img_augmentation(np.expand_dims(image.numpy(), axis=0))
- aug_img = np.array(aug_img)
- plt.imshow(aug_img[0].astype("uint8"))
- plt.title("{}".format(format_label(label)))
- plt.axis("off")
+first_image, first_label = preview_train[0]
+
+for i in range(9):
+ ax = plt.subplot(3, 3, i + 1)
+ aug_img = img_augmentation(np.expand_dims(np.array(first_image), axis=0))
+ aug_img = np.array(aug_img)
+ plt.imshow(aug_img[0].astype("uint8"))
+ plt.title("{}".format(format_label(first_label)))
+ plt.axis("off")
"""
### Prepare inputs
Once we verify the input data and augmentation are working correctly,
-we prepare dataset for training. The input data are resized to uniform
-`IMG_SIZE`. The labels are put into one-hot
-(a.k.a. categorical) encoding. The dataset is batched.
-
-Note: `prefetch` and `AUTOTUNE` may in some situation improve
-performance, but depends on environment and the specific dataset used.
-See this [guide](https://www.tensorflow.org/guide/data_performance)
-for more information on data pipeline performance.
-"""
-
-
-# One-hot / categorical encoding
-def input_preprocess_train(image, label):
- image = img_augmentation(image)
- label = tf.one_hot(label, NUM_CLASSES)
- return image, label
-
-
-def input_preprocess_test(image, label):
- label = tf.one_hot(label, NUM_CLASSES)
- return image, label
-
-
-ds_train = ds_train.map(input_preprocess_train, num_parallel_calls=tf.data.AUTOTUNE)
-ds_train = ds_train.batch(batch_size=BATCH_SIZE, drop_remainder=True)
-ds_train = ds_train.prefetch(tf.data.AUTOTUNE)
+we prepare backend-agnostic datasets for training.
+
+The input images are resized to uniform `IMG_SIZE`, labels are converted to one-hot
+(categorical) encoding, and batches are produced by `keras.utils.PyDataset`.
+
+Compared to the original `tf.data` version, this Keras 3 setup is backend-agnostic
+and works seamlessly across TensorFlow, JAX, and PyTorch backends.
+"""
+
+
+class StanfordDogsDataset(keras.utils.PyDataset):
+ def __init__(
+ self,
+ image_paths,
+ labels,
+ num_classes,
+ img_size,
+ batch_size,
+ augment=False,
+ shuffle=False,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self.image_paths = image_paths
+ self.labels = labels
+ self.num_classes = num_classes
+ self.img_size = img_size
+ self.batch_size = batch_size
+ self.augment = augment
+ self.shuffle = shuffle
+ self.indices = np.arange(len(labels))
+ self.on_epoch_end()
+
+ def __len__(self):
+ # Match previous drop_remainder=True behavior.
+ return len(self.indices) // self.batch_size
+
+ def on_epoch_end(self):
+ if self.shuffle:
+ np.random.shuffle(self.indices)
+
+ def __getitem__(self, idx):
+ batch_indices = self.indices[
+ idx * self.batch_size : (idx + 1) * self.batch_size
+ ]
+
+ # Load images lazily from disk
+ batch_images = []
+ for i in batch_indices:
+ img = Image.open(self.image_paths[i]).convert("RGB")
+ img_array = np.array(img, dtype="float32")
+ img_resized = keras.ops.image.resize(
+ img_array, (self.img_size, self.img_size)
+ )
+ batch_images.append(np.array(img_resized))
+ batch_images = np.stack(batch_images)
+
+ if self.augment:
+ batch_images = np.array(img_augmentation(batch_images))
+
+ batch_labels = np.array(
+ keras.ops.one_hot(self.labels[batch_indices], self.num_classes)
+ )
+
+ return batch_images, batch_labels
+
+
+ds_train = StanfordDogsDataset(
+ train_image_paths,
+ train_labels,
+ num_classes=NUM_CLASSES,
+ img_size=IMG_SIZE,
+ batch_size=BATCH_SIZE,
+ augment=True,
+ shuffle=True,
+ workers=2,
+ use_multiprocessing=False,
+)
-ds_test = ds_test.map(input_preprocess_test, num_parallel_calls=tf.data.AUTOTUNE)
-ds_test = ds_test.batch(batch_size=BATCH_SIZE, drop_remainder=True)
+ds_test = StanfordDogsDataset(
+ test_image_paths,
+ test_labels,
+ num_classes=NUM_CLASSES,
+ img_size=IMG_SIZE,
+ batch_size=BATCH_SIZE,
+ augment=False,
+ shuffle=False,
+)
"""
## Training a model from scratch
-We build an EfficientNetB0 with 120 output classes, that is initialized from scratch:
+We build an EfficientNetB0 with 120 output classes, initialized from scratch
+(no pretrained weights).
-Note: the accuracy will increase very slowly and may overfit.
+**Note:** Training from scratch typically shows slower convergence and may overfit
+on smaller datasets like Stanford Dogs.
"""
model = EfficientNetB0(
@@ -290,13 +437,13 @@ def input_preprocess_test(image, label):
"""
-Training the model is relatively fast. This might make it sounds easy to simply train EfficientNet on any
-dataset wanted from scratch. However, training EfficientNet on smaller datasets,
+Training the model is relatively fast (a few minutes per epoch on modern hardware).
+However, training EfficientNet on smaller datasets,
especially those with lower resolution like CIFAR-100, faces the significant challenge of
overfitting.
-Hence training from scratch requires very careful choice of hyperparameters and is
-difficult to find suitable regularization. It would also be much more demanding in resources.
+Training from scratch requires very careful choice of hyperparameters and
+suitable regularization. It is also much more demanding in computational resources.
Plotting the training and validation accuracy
makes it clear that validation accuracy stagnates at a low value.
"""
@@ -317,10 +464,11 @@ def plot_hist(hist):
plot_hist(hist)
"""
-## Transfer learning from pre-trained weights
+## Transfer learning from pretrained weights
-Here we initialize the model with pre-trained ImageNet weights,
-and we fine-tune it on our own dataset.
+Here we initialize the model with pretrained ImageNet weights
+and fine-tune it on our own dataset. This is the recommended approach
+for most applications.
"""
@@ -349,15 +497,15 @@ def build_model(num_classes):
"""
-The first step to transfer learning is to freeze all layers and train only the top
+The first step in transfer learning is to freeze all base layers and train only the top
layers. For this step, a relatively large learning rate (1e-2) can be used.
-Note that validation accuracy and loss will usually be better than training
+
+**Note:** Validation accuracy and loss will usually be better than training
accuracy and loss. This is because the regularization is strong, which only
suppresses training-time metrics.
-Note that the convergence may take up to 50 epochs depending on choice of learning rate.
-If image augmentation layers were not
-applied, the validation accuracy may only reach ~60%.
+The convergence may take up to 50 epochs depending on the choice of learning rate.
+If image augmentation layers were not applied, the validation accuracy may only reach ~60%.
"""
model = build_model(num_classes=NUM_CLASSES)
@@ -367,14 +515,18 @@ def build_model(num_classes):
plot_hist(hist)
"""
-The second step is to unfreeze a number of layers and fit the model using smaller
-learning rate. In this example we show unfreezing all layers, but depending on
-specific dataset it may be desireble to only unfreeze a fraction of all layers.
+The second step is to unfreeze a number of layers and fine-tune the model using a smaller
+learning rate. In this example we unfreeze the last 20 layers, but depending on the
+specific dataset it may be desirable to only unfreeze a fraction of all layers.
+
+**Advanced usage:** The `unfreeze_model()` function also supports unfreezing by block name
+(e.g., `unfreeze_model(model, layers_to_unfreeze="block7")`) to respect EfficientNet's
+residual block boundaries. See the "Tips for fine-tuning EfficientNet" section below
+for why this matters.
-When the feature extraction with
-pretrained model works good enough, this step would give a very limited gain on
-validation accuracy. In our case we only see a small improvement,
-as ImageNet pretraining already exposed the model to a good amount of dogs.
+When feature extraction with the pretrained model works well enough, this step provides
+only a limited gain in validation accuracy. In our case we only see a small improvement,
+as ImageNet pretraining already exposed the model to a good amount of dog images.
On the other hand, when we use pretrained weights on a dataset that is more different
from ImageNet, this fine-tuning step can be crucial as the feature extractor also
@@ -382,7 +534,7 @@ def build_model(num_classes):
if choosing CIFAR-100 dataset instead, where fine-tuning boosts validation accuracy
by about 10% to pass 80% on `EfficientNetB0`.
-A side note on freezing/unfreezing models: setting `trainable` of a `Model` will
+**Note on freezing/unfreezing models:** Setting `trainable` of a `Model` will
simultaneously set all layers belonging to the `Model` to the same `trainable`
attribute. Each layer is trainable only if both the layer itself and the model
containing it are trainable. Hence when we need to partially freeze/unfreeze
@@ -391,53 +543,125 @@ def build_model(num_classes):
"""
-def unfreeze_model(model):
- # We unfreeze the top 20 layers while leaving BatchNorm layers frozen
- for layer in model.layers[-20:]:
+def unfreeze_model(
+ model,
+ layers_to_unfreeze=20,
+ learning_rate=1e-5,
+ loss="categorical_crossentropy",
+ metrics=None,
+):
+ """Unfreeze part of `model` and recompile it for fine-tuning.
+
+ Args:
+ model: A `keras.Model` instance to unfreeze in place.
+ layers_to_unfreeze: Either an `int` giving the number of layers,
+ counted from the end of the base model's layers, to unfreeze, or a `str`
+ substring to match against layer names -- the first matching
+ layer and every layer after it (in the base model's layers order) are
+ unfrozen. Use a string like `"block7"` to respect EfficientNet's
+ residual block boundaries instead of an arbitrary layer count.
+ Defaults to `20`.
+ learning_rate: Learning rate for the fine-tuning `Adam` optimizer.
+ Defaults to `1e-5`.
+ loss: Loss function passed to `model.compile()`. Defaults to
+ `"categorical_crossentropy"`.
+ metrics: List of metrics passed to `model.compile()`. Defaults to
+ `["accuracy"]`.
+
+ Returns:
+ `model`, with the selected layers unfrozen (except
+ `BatchNormalization` layers, which are always kept frozen) and
+ recompiled with the new optimizer/loss/metrics.
+ """
+ if metrics is None:
+ metrics = ["accuracy"]
+
+ # Access the nested EfficientNet base model by finding the first layer
+ # with 'efficientnet' in its name (case-insensitive)
+ base_model = None
+ for layer in model.layers:
+ if "efficientnet" in layer.name.lower():
+ base_model = layer
+ break
+ if base_model is None:
+ raise ValueError(
+ "Could not find EfficientNet base model in the model. "
+ "Expected a layer with 'efficientnet' in its name."
+ )
+ base_model.trainable = True
+
+ # First, freeze all layers in the base model
+ for layer in base_model.layers:
+ layer.trainable = False
+
+ if isinstance(layers_to_unfreeze, str):
+ unfreeze_from = None
+ for i, layer in enumerate(base_model.layers):
+ if layers_to_unfreeze in layer.name:
+ unfreeze_from = i
+ break
+ if unfreeze_from is None:
+ raise ValueError(f"No layer name contains {layers_to_unfreeze!r}.")
+ layers_to_process = base_model.layers[unfreeze_from:]
+ elif isinstance(layers_to_unfreeze, int):
+ if layers_to_unfreeze <= 0:
+ raise ValueError(
+ f"layers_to_unfreeze must be > 0, got {layers_to_unfreeze}"
+ )
+ n_layers_to_unfreeze = min(layers_to_unfreeze, len(base_model.layers))
+ layers_to_process = base_model.layers[-n_layers_to_unfreeze:]
+ else:
+ raise TypeError(
+ "layers_to_unfreeze must be an int or str, received: "
+ f"{type(layers_to_unfreeze)}"
+ )
+
+ # We keep BatchNorm layers frozen -- see "Tips for fine-tuning
+ # EfficientNet" in the tutorial for why.
+ for layer in layers_to_process:
if not isinstance(layer, layers.BatchNormalization):
layer.trainable = True
- optimizer = keras.optimizers.Adam(learning_rate=1e-5)
- model.compile(
- optimizer=optimizer, loss="categorical_crossentropy", metrics=["accuracy"]
- )
+ optimizer = keras.optimizers.Adam(learning_rate=learning_rate)
+ model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
+ return model
-unfreeze_model(model)
+model = unfreeze_model(model)
epochs = 4 # @param {type: "slider", min:4, max:10}
hist = model.fit(ds_train, epochs=epochs, validation_data=ds_test)
plot_hist(hist)
"""
-### Tips for fine tuning EfficientNet
+### Tips for fine-tuning EfficientNet
-On unfreezing layers:
+**On unfreezing layers:**
- The `BatchNormalization` layers need to be kept frozen
([more details](https://keras.io/guides/transfer_learning/)).
If they are also turned to trainable, the
first epoch after unfreezing will significantly reduce accuracy.
-- In some cases it may be beneficial to open up only a portion of layers instead of
-unfreezing all. This will make fine tuning much faster when going to larger models like
+- In some cases it may be beneficial to unfreeze only a portion of layers instead of
+unfreezing all. This will make fine-tuning much faster when going to larger models like
B7.
- Each block needs to be all turned on or off. This is because the architecture includes
a shortcut from the first layer to the last layer for each block. Not respecting blocks
also significantly harms the final performance.
-Some other tips for utilizing EfficientNet:
+**Some other tips for utilizing EfficientNet:**
- Larger variants of EfficientNet do not guarantee improved performance, especially for
-tasks with less data or fewer classes. In such a case, the larger variant of EfficientNet
+tasks with less data or fewer classes. In such a case, the larger the variant of EfficientNet
chosen, the harder it is to tune hyperparameters.
- EMA (Exponential Moving Average) is very helpful in training EfficientNet from scratch,
but not so much for transfer learning.
- Do not use the RMSprop setup as in the original paper for transfer learning. The
momentum and learning rate are too high for transfer learning. It will easily corrupt the
-pretrained weight and blow up the loss. A quick check is to see if loss (as categorical
+pretrained weights and blow up the loss. A quick check is to see if loss (as categorical
cross entropy) is getting significantly larger than log(NUM_CLASSES) after the same
epoch. If so, the initial learning rate/momentum is too high.
-- Smaller batch size benefit validation accuracy, possibly due to effectively providing
+- Smaller batch sizes benefit validation accuracy, possibly due to effectively providing
regularization.
"""
diff --git a/examples/vision/ipynb/image_classification_efficientnet_fine_tuning.ipynb b/examples/vision/ipynb/image_classification_efficientnet_fine_tuning.ipynb
index 69b6a60ca5..4a073567d8 100644
--- a/examples/vision/ipynb/image_classification_efficientnet_fine_tuning.ipynb
+++ b/examples/vision/ipynb/image_classification_efficientnet_fine_tuning.ipynb
@@ -10,7 +10,7 @@
"\n",
"**Author:** [Yixing Fu](https://github.com/yixingfu)
\n",
"**Date created:** 2020/06/30
\n",
- "**Last modified:** 2023/07/10
\n",
+ "**Last modified:** 2026/07/13
\n",
"**Description:** Use EfficientNet with weights pre-trained on imagenet for Stanford Dogs classification."
]
},
@@ -36,12 +36,6 @@
"efficiency-oriented base model (B0) to surpass models at every scale, while avoiding\n",
"extensive grid-search of hyperparameters.\n",
"\n",
- "A summary of the latest updates on the model is available at\n",
- "[here](https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet), where various\n",
- "augmentation schemes and semi-supervised learning approaches are applied to further\n",
- "improve the imagenet performance of the models. These extensions of the model can be used\n",
- "by updating weights without changing model architecture.\n",
- "\n",
"## B0 to B7 variants of EfficientNet\n",
"\n",
"*(This section provides some details on \"compound scaling\", and can be skipped\n",
@@ -68,7 +62,7 @@
"are hand-picked and proven to produce good results, though they may be significantly\n",
"off from the compound scaling formula.\n",
"Therefore, the keras implementation (detailed below) only provide these 8 models, B0 to B7,\n",
- "instead of allowing arbitray choice of width / depth / resolution parameters.\n",
+ "instead of allowing arbitrary choice of width / depth / resolution parameters.\n",
"\n",
"## Keras implementation of EfficientNet\n",
"\n",
@@ -76,7 +70,7 @@
"use EfficientNetB0 for classifying 1000 classes of images from ImageNet, run:\n",
"\n",
"```python\n",
- "from tensorflow.keras.applications import EfficientNetB0\n",
+ "from keras.applications import EfficientNetB0\n",
"model = EfficientNetB0(weights='imagenet')\n",
"```\n",
"\n",
@@ -146,17 +140,21 @@
},
"outputs": [],
"source": [
+ "import tarfile\n",
+ "import urllib.request\n",
+ "from pathlib import Path\n",
"import numpy as np\n",
- "import tensorflow_datasets as tfds\n",
- "import tensorflow as tf # For tf.data\n",
"import matplotlib.pyplot as plt\n",
+ "from PIL import Image\n",
+ "import scipy.io\n",
"import keras\n",
"from keras import layers\n",
"from keras.applications import EfficientNetB0\n",
"\n",
"# IMG_SIZE is determined by EfficientNet model choice\n",
"IMG_SIZE = 224\n",
- "BATCH_SIZE = 64"
+ "BATCH_SIZE = 64\n",
+ ""
]
},
{
@@ -167,22 +165,17 @@
"source": [
"### Loading data\n",
"\n",
- "Here we load data from [tensorflow_datasets](https://www.tensorflow.org/datasets)\n",
- "(hereafter TFDS).\n",
- "Stanford Dogs dataset is provided in\n",
- "TFDS as [stanford_dogs](https://www.tensorflow.org/datasets/catalog/stanford_dogs).\n",
- "It features 20,580 images that belong to 120 classes of dog breeds\n",
+ "We download the Stanford Dogs dataset directly from Stanford's servers using the\n",
+ "built-in `urllib` and `tarfile` modules.\n",
+ "The dataset contains 20,580 images belonging to 120 classes of dog breeds\n",
"(12,000 for training and 8,580 for testing).\n",
"\n",
- "By simply changing `dataset_name` below, you may also try this notebook for\n",
- "other datasets in TFDS such as\n",
- "[cifar10](https://www.tensorflow.org/datasets/catalog/cifar10),\n",
- "[cifar100](https://www.tensorflow.org/datasets/catalog/cifar100),\n",
- "[food101](https://www.tensorflow.org/datasets/catalog/food101),\n",
- "etc. When the images are much smaller than the size of EfficientNet input,\n",
- "we can simply upsample the input images. It has been shown in\n",
- "[Tan and Le, 2019](https://arxiv.org/abs/1905.11946) that transfer learning\n",
- "result is better for increased resolution even if input images remain small."
+ "The dataset is downloaded, extracted, and loaded into lists of NumPy arrays.\n",
+ "Images have variable dimensions and will be resized to a uniform size in the data pipeline.\n",
+ "\n",
+ "**Note:** This direct download approach eliminates dependency conflicts that can\n",
+ "occur with `tensorflow_datasets` in some environments (particularly Google Colab\n",
+ "with protobuf version incompatibilities)."
]
},
{
@@ -193,11 +186,66 @@
},
"outputs": [],
"source": [
- "dataset_name = \"stanford_dogs\"\n",
- "(ds_train, ds_test), ds_info = tfds.load(\n",
- " dataset_name, split=[\"train\", \"test\"], with_info=True, as_supervised=True\n",
+ "dataset_url = \"http://vision.stanford.edu/aditya86/ImageNetDogs/images.tar\"\n",
+ "lists_url = \"http://vision.stanford.edu/aditya86/ImageNetDogs/lists.tar\"\n",
+ "data_dir = Path(\"./stanford_dogs_data\")\n",
+ "data_dir.mkdir(exist_ok=True)\n",
+ "\n",
+ "\n",
+ "def download_and_extract(url, extract_to):\n",
+ " filename = url.split(\"/\")[-1]\n",
+ " filepath = data_dir / filename\n",
+ " if not filepath.exists():\n",
+ " print(f\"Downloading {filename}...\")\n",
+ " urllib.request.urlretrieve(url, filepath)\n",
+ " print(f\"Extracting {filename}...\")\n",
+ " with tarfile.open(filepath, \"r\") as tar:\n",
+ " tar.extractall(extract_to, filter=\"data\")\n",
+ " return extract_to\n",
+ "\n",
+ "\n",
+ "# Download dataset\n",
+ "images_dir = download_and_extract(dataset_url, data_dir)\n",
+ "lists_dir = download_and_extract(lists_url, data_dir)\n",
+ "\n",
+ "\n",
+ "# Parse train/test splits\n",
+ "def load_file_list(filepath):\n",
+ " mat = scipy.io.loadmat(filepath)\n",
+ " return [item[0][0] for item in mat[\"file_list\"]]\n",
+ "\n",
+ "\n",
+ "train_files = load_file_list(data_dir / \"train_list.mat\")\n",
+ "test_files = load_file_list(data_dir / \"test_list.mat\")\n",
+ "\n",
+ "# Build class name mapping\n",
+ "all_files = train_files + test_files\n",
+ "class_names = sorted(set([f.split(\"/\")[0] for f in all_files]))\n",
+ "class_to_idx = {name: idx for idx, name in enumerate(class_names)}\n",
+ "NUM_CLASSES = len(class_names)\n",
+ "\n",
+ "print(\n",
+ " f\"Found {NUM_CLASSES} classes, {len(train_files)} training images, {len(test_files)} test images\"\n",
")\n",
- "NUM_CLASSES = ds_info.features[\"label\"].num_classes"
+ "\n",
+ "\n",
+ "# Prepare image paths and labels (lazy loading - no images loaded into memory yet)\n",
+ "def prepare_paths_and_labels(file_list, base_dir):\n",
+ " image_paths, labels = [], []\n",
+ " for file_path in file_list:\n",
+ " class_name = file_path.split(\"/\")[0]\n",
+ " img_path = base_dir / \"Images\" / file_path\n",
+ " if img_path.exists():\n",
+ " image_paths.append(str(img_path))\n",
+ " labels.append(class_to_idx[class_name])\n",
+ " return image_paths, np.array(labels)\n",
+ "\n",
+ "\n",
+ "print(\"Preparing dataset paths...\")\n",
+ "train_image_paths, train_labels = prepare_paths_and_labels(train_files, data_dir)\n",
+ "test_image_paths, test_labels = prepare_paths_and_labels(test_files, data_dir)\n",
+ "print(f\"Found {len(train_image_paths)} train and {len(test_image_paths)} test images\")\n",
+ ""
]
},
{
@@ -206,9 +254,13 @@
"colab_type": "text"
},
"source": [
- "When the dataset include images with various size, we need to resize them into a\n",
- "shared size. The Stanford Dogs dataset includes only images at least 200x200\n",
- "pixels in size. Here we resize the images to the input size needed for EfficientNet."
+ "Each image can have a different shape, so we resize them to a shared input size\n",
+ "for EfficientNet. In Keras 3, we do this in a backend-agnostic `PyDataset` pipeline\n",
+ "using `keras.ops.image.resize`, which works across TensorFlow, JAX, and PyTorch backends,\n",
+ "rather than the TensorFlow-specific `tf.data` mapping steps.\n",
+ "\n",
+ "Images are loaded lazily from disk in `__getitem__` to avoid loading all 20,580 images\n",
+ "into memory at once, which would consume several gigabytes of RAM and cause OOM issues."
]
},
{
@@ -219,9 +271,43 @@
},
"outputs": [],
"source": [
- "size = (IMG_SIZE, IMG_SIZE)\n",
- "ds_train = ds_train.map(lambda image, label: (tf.image.resize(image, size), label))\n",
- "ds_test = ds_test.map(lambda image, label: (tf.image.resize(image, size), label))"
+ "\n",
+ "class ResizeOnlyDataset(keras.utils.PyDataset):\n",
+ " def __init__(self, image_paths, labels, img_size, batch_size=1, **kwargs):\n",
+ " super().__init__(**kwargs)\n",
+ " self.image_paths = image_paths\n",
+ " self.labels = labels\n",
+ " self.img_size = img_size\n",
+ " self.batch_size = batch_size\n",
+ " self.indices = np.arange(len(labels))\n",
+ "\n",
+ " def __len__(self):\n",
+ " return int(np.ceil(len(self.labels) / self.batch_size))\n",
+ "\n",
+ " def __getitem__(self, idx):\n",
+ " batch_indices = self.indices[\n",
+ " idx * self.batch_size : (idx + 1) * self.batch_size\n",
+ " ]\n",
+ " batch_images = []\n",
+ " for i in batch_indices:\n",
+ " img = Image.open(self.image_paths[i]).convert(\"RGB\")\n",
+ " img_array = np.array(img, dtype=\"float32\")\n",
+ " img_resized = keras.ops.image.resize(\n",
+ " img_array, (self.img_size, self.img_size)\n",
+ " )\n",
+ " batch_images.append(np.array(img_resized))\n",
+ " batch_images = np.stack(batch_images)\n",
+ " batch_labels = self.labels[batch_indices]\n",
+ " if self.batch_size == 1:\n",
+ " return batch_images[0], batch_labels[0]\n",
+ " return batch_images, batch_labels\n",
+ "\n",
+ "\n",
+ "# Preview stream with resized images for visualization below\n",
+ "preview_train = ResizeOnlyDataset(\n",
+ " train_image_paths, train_labels, IMG_SIZE, batch_size=1\n",
+ ")\n",
+ "preview_test = ResizeOnlyDataset(test_image_paths, test_labels, IMG_SIZE, batch_size=1)"
]
},
{
@@ -243,17 +329,20 @@
},
"outputs": [],
"source": [
+ "\n",
"def format_label(label):\n",
- " string_label = label_info.int2str(label)\n",
- " return string_label.split(\"-\")[1]\n",
+ " class_name = class_names[int(label)]\n",
+ " return class_name.split(\"-\")[1] # Extract breed name from \"n02085620-Chihuahua\"\n",
"\n",
"\n",
- "label_info = ds_info.features[\"label\"]\n",
- "for i, (image, label) in enumerate(ds_train.take(9)):\n",
+ "for i, (image, label) in enumerate(preview_train):\n",
+ " if i >= 9:\n",
+ " break\n",
" ax = plt.subplot(3, 3, i + 1)\n",
- " plt.imshow(image.numpy().astype(\"uint8\"))\n",
+ " plt.imshow(np.array(image).astype(\"uint8\"))\n",
" plt.title(\"{}\".format(format_label(label)))\n",
- " plt.axis(\"off\")"
+ " plt.axis(\"off\")\n",
+ ""
]
},
{
@@ -264,7 +353,8 @@
"source": [
"### Data augmentation\n",
"\n",
- "We can use the preprocessing layers APIs for image augmentation."
+ "We can use Keras preprocessing layers for image augmentation.\n",
+ "These layers are backend-agnostic and can be used during both training and inference."
]
},
{
@@ -286,7 +376,8 @@
"def img_augmentation(images):\n",
" for layer in img_augmentation_layers:\n",
" images = layer(images)\n",
- " return images"
+ " return images\n",
+ ""
]
},
{
@@ -295,11 +386,10 @@
"colab_type": "text"
},
"source": [
- "This `Sequential` model object can be used both as a part of\n",
- "the model we later build, and as a function to preprocess\n",
- "data before feeding into the model. Using them as function makes\n",
- "it easy to visualize the augmented images. Here we plot 9 examples\n",
- "of augmentation result of a given figure."
+ "The `img_augmentation` function can be used both as a part of the model\n",
+ "we later build, and as a standalone function to preprocess data before feeding\n",
+ "into the model. Using it as a function makes it easy to visualize the augmentation\n",
+ "results. Here we plot 9 examples of augmentation applied to a single image."
]
},
{
@@ -310,14 +400,16 @@
},
"outputs": [],
"source": [
- "for image, label in ds_train.take(1):\n",
- " for i in range(9):\n",
- " ax = plt.subplot(3, 3, i + 1)\n",
- " aug_img = img_augmentation(np.expand_dims(image.numpy(), axis=0))\n",
- " aug_img = np.array(aug_img)\n",
- " plt.imshow(aug_img[0].astype(\"uint8\"))\n",
- " plt.title(\"{}\".format(format_label(label)))\n",
- " plt.axis(\"off\")"
+ "first_image, first_label = preview_train[0]\n",
+ "\n",
+ "for i in range(9):\n",
+ " ax = plt.subplot(3, 3, i + 1)\n",
+ " aug_img = img_augmentation(np.expand_dims(np.array(first_image), axis=0))\n",
+ " aug_img = np.array(aug_img)\n",
+ " plt.imshow(aug_img[0].astype(\"uint8\"))\n",
+ " plt.title(\"{}\".format(format_label(first_label)))\n",
+ " plt.axis(\"off\")\n",
+ ""
]
},
{
@@ -329,14 +421,13 @@
"### Prepare inputs\n",
"\n",
"Once we verify the input data and augmentation are working correctly,\n",
- "we prepare dataset for training. The input data are resized to uniform\n",
- "`IMG_SIZE`. The labels are put into one-hot\n",
- "(a.k.a. categorical) encoding. The dataset is batched.\n",
- "\n",
- "Note: `prefetch` and `AUTOTUNE` may in some situation improve\n",
- "performance, but depends on environment and the specific dataset used.\n",
- "See this [guide](https://www.tensorflow.org/guide/data_performance)\n",
- "for more information on data pipeline performance."
+ "we prepare backend-agnostic datasets for training.\n",
+ "\n",
+ "The input images are resized to uniform `IMG_SIZE`, labels are converted to one-hot\n",
+ "(categorical) encoding, and batches are produced by `keras.utils.PyDataset`.\n",
+ "\n",
+ "Compared to the original `tf.data` version, this Keras 3 setup is backend-agnostic\n",
+ "and works seamlessly across TensorFlow, JAX, and PyTorch backends."
]
},
{
@@ -347,24 +438,86 @@
},
"outputs": [],
"source": [
- "# One-hot / categorical encoding\n",
- "def input_preprocess_train(image, label):\n",
- " image = img_augmentation(image)\n",
- " label = tf.one_hot(label, NUM_CLASSES)\n",
- " return image, label\n",
- "\n",
"\n",
- "def input_preprocess_test(image, label):\n",
- " label = tf.one_hot(label, NUM_CLASSES)\n",
- " return image, label\n",
- "\n",
- "\n",
- "ds_train = ds_train.map(input_preprocess_train, num_parallel_calls=tf.data.AUTOTUNE)\n",
- "ds_train = ds_train.batch(batch_size=BATCH_SIZE, drop_remainder=True)\n",
- "ds_train = ds_train.prefetch(tf.data.AUTOTUNE)\n",
+ "class StanfordDogsDataset(keras.utils.PyDataset):\n",
+ " def __init__(\n",
+ " self,\n",
+ " image_paths,\n",
+ " labels,\n",
+ " num_classes,\n",
+ " img_size,\n",
+ " batch_size,\n",
+ " augment=False,\n",
+ " shuffle=False,\n",
+ " **kwargs,\n",
+ " ):\n",
+ " super().__init__(**kwargs)\n",
+ " self.image_paths = image_paths\n",
+ " self.labels = labels\n",
+ " self.num_classes = num_classes\n",
+ " self.img_size = img_size\n",
+ " self.batch_size = batch_size\n",
+ " self.augment = augment\n",
+ " self.shuffle = shuffle\n",
+ " self.indices = np.arange(len(labels))\n",
+ " self.on_epoch_end()\n",
+ "\n",
+ " def __len__(self):\n",
+ " # Match previous drop_remainder=True behavior.\n",
+ " return len(self.indices) // self.batch_size\n",
+ "\n",
+ " def on_epoch_end(self):\n",
+ " if self.shuffle:\n",
+ " np.random.shuffle(self.indices)\n",
+ "\n",
+ " def __getitem__(self, idx):\n",
+ " batch_indices = self.indices[\n",
+ " idx * self.batch_size : (idx + 1) * self.batch_size\n",
+ " ]\n",
+ "\n",
+ " # Load images lazily from disk\n",
+ " batch_images = []\n",
+ " for i in batch_indices:\n",
+ " img = Image.open(self.image_paths[i]).convert(\"RGB\")\n",
+ " img_array = np.array(img, dtype=\"float32\")\n",
+ " img_resized = keras.ops.image.resize(\n",
+ " img_array, (self.img_size, self.img_size)\n",
+ " )\n",
+ " batch_images.append(np.array(img_resized))\n",
+ " batch_images = np.stack(batch_images)\n",
+ "\n",
+ " if self.augment:\n",
+ " batch_images = np.array(img_augmentation(batch_images))\n",
+ "\n",
+ " batch_labels = np.array(\n",
+ " keras.ops.one_hot(self.labels[batch_indices], self.num_classes)\n",
+ " )\n",
+ "\n",
+ " return batch_images, batch_labels\n",
+ "\n",
+ "\n",
+ "ds_train = StanfordDogsDataset(\n",
+ " train_image_paths,\n",
+ " train_labels,\n",
+ " num_classes=NUM_CLASSES,\n",
+ " img_size=IMG_SIZE,\n",
+ " batch_size=BATCH_SIZE,\n",
+ " augment=True,\n",
+ " shuffle=True,\n",
+ " workers=2,\n",
+ " use_multiprocessing=False,\n",
+ ")\n",
"\n",
- "ds_test = ds_test.map(input_preprocess_test, num_parallel_calls=tf.data.AUTOTUNE)\n",
- "ds_test = ds_test.batch(batch_size=BATCH_SIZE, drop_remainder=True)"
+ "ds_test = StanfordDogsDataset(\n",
+ " test_image_paths,\n",
+ " test_labels,\n",
+ " num_classes=NUM_CLASSES,\n",
+ " img_size=IMG_SIZE,\n",
+ " batch_size=BATCH_SIZE,\n",
+ " augment=False,\n",
+ " shuffle=False,\n",
+ ")\n",
+ ""
]
},
{
@@ -375,9 +528,11 @@
"source": [
"## Training a model from scratch\n",
"\n",
- "We build an EfficientNetB0 with 120 output classes, that is initialized from scratch:\n",
+ "We build an EfficientNetB0 with 120 output classes, initialized from scratch\n",
+ "(no pretrained weights).\n",
"\n",
- "Note: the accuracy will increase very slowly and may overfit."
+ "**Note:** Training from scratch typically shows slower convergence and may overfit\n",
+ "on smaller datasets like Stanford Dogs."
]
},
{
@@ -399,7 +554,8 @@
"model.summary()\n",
"\n",
"epochs = 40 # @param {type: \"slider\", min:10, max:100}\n",
- "hist = model.fit(ds_train, epochs=epochs, validation_data=ds_test)"
+ "hist = model.fit(ds_train, epochs=epochs, validation_data=ds_test)\n",
+ ""
]
},
{
@@ -408,13 +564,13 @@
"colab_type": "text"
},
"source": [
- "Training the model is relatively fast. This might make it sounds easy to simply train EfficientNet on any\n",
- "dataset wanted from scratch. However, training EfficientNet on smaller datasets,\n",
+ "Training the model is relatively fast (a few minutes per epoch on modern hardware).\n",
+ "However, training EfficientNet on smaller datasets,\n",
"especially those with lower resolution like CIFAR-100, faces the significant challenge of\n",
"overfitting.\n",
"\n",
- "Hence training from scratch requires very careful choice of hyperparameters and is\n",
- "difficult to find suitable regularization. It would also be much more demanding in resources.\n",
+ "Training from scratch requires very careful choice of hyperparameters and\n",
+ "suitable regularization. It is also much more demanding in computational resources.\n",
"Plotting the training and validation accuracy\n",
"makes it clear that validation accuracy stagnates at a low value."
]
@@ -449,10 +605,11 @@
"colab_type": "text"
},
"source": [
- "## Transfer learning from pre-trained weights\n",
+ "## Transfer learning from pretrained weights\n",
"\n",
- "Here we initialize the model with pre-trained ImageNet weights,\n",
- "and we fine-tune it on our own dataset."
+ "Here we initialize the model with pretrained ImageNet weights\n",
+ "and fine-tune it on our own dataset. This is the recommended approach\n",
+ "for most applications."
]
},
{
@@ -463,6 +620,7 @@
},
"outputs": [],
"source": [
+ "\n",
"def build_model(num_classes):\n",
" inputs = layers.Input(shape=(IMG_SIZE, IMG_SIZE, 3))\n",
" model = EfficientNetB0(include_top=False, input_tensor=inputs, weights=\"imagenet\")\n",
@@ -484,7 +642,8 @@
" model.compile(\n",
" optimizer=optimizer, loss=\"categorical_crossentropy\", metrics=[\"accuracy\"]\n",
" )\n",
- " return model"
+ " return model\n",
+ ""
]
},
{
@@ -493,15 +652,15 @@
"colab_type": "text"
},
"source": [
- "The first step to transfer learning is to freeze all layers and train only the top\n",
+ "The first step in transfer learning is to freeze all base layers and train only the top\n",
"layers. For this step, a relatively large learning rate (1e-2) can be used.\n",
- "Note that validation accuracy and loss will usually be better than training\n",
+ "\n",
+ "**Note:** Validation accuracy and loss will usually be better than training\n",
"accuracy and loss. This is because the regularization is strong, which only\n",
"suppresses training-time metrics.\n",
"\n",
- "Note that the convergence may take up to 50 epochs depending on choice of learning rate.\n",
- "If image augmentation layers were not\n",
- "applied, the validation accuracy may only reach ~60%."
+ "The convergence may take up to 50 epochs depending on the choice of learning rate.\n",
+ "If image augmentation layers were not applied, the validation accuracy may only reach ~60%."
]
},
{
@@ -525,14 +684,18 @@
"colab_type": "text"
},
"source": [
- "The second step is to unfreeze a number of layers and fit the model using smaller\n",
- "learning rate. In this example we show unfreezing all layers, but depending on\n",
- "specific dataset it may be desireble to only unfreeze a fraction of all layers.\n",
+ "The second step is to unfreeze a number of layers and fine-tune the model using a smaller\n",
+ "learning rate. In this example we unfreeze the last 20 layers, but depending on the\n",
+ "specific dataset it may be desirable to only unfreeze a fraction of all layers.\n",
+ "\n",
+ "**Advanced usage:** The `unfreeze_model()` function also supports unfreezing by block name\n",
+ "(e.g., `unfreeze_model(model, layers_to_unfreeze=\"block7\")`) to respect EfficientNet's\n",
+ "residual block boundaries. See the \"Tips for fine-tuning EfficientNet\" section below\n",
+ "for why this matters.\n",
"\n",
- "When the feature extraction with\n",
- "pretrained model works good enough, this step would give a very limited gain on\n",
- "validation accuracy. In our case we only see a small improvement,\n",
- "as ImageNet pretraining already exposed the model to a good amount of dogs.\n",
+ "When feature extraction with the pretrained model works well enough, this step provides\n",
+ "only a limited gain in validation accuracy. In our case we only see a small improvement,\n",
+ "as ImageNet pretraining already exposed the model to a good amount of dog images.\n",
"\n",
"On the other hand, when we use pretrained weights on a dataset that is more different\n",
"from ImageNet, this fine-tuning step can be crucial as the feature extractor also\n",
@@ -540,7 +703,7 @@
"if choosing CIFAR-100 dataset instead, where fine-tuning boosts validation accuracy\n",
"by about 10% to pass 80% on `EfficientNetB0`.\n",
"\n",
- "A side note on freezing/unfreezing models: setting `trainable` of a `Model` will\n",
+ "**Note on freezing/unfreezing models:** Setting `trainable` of a `Model` will\n",
"simultaneously set all layers belonging to the `Model` to the same `trainable`\n",
"attribute. Each layer is trainable only if both the layer itself and the model\n",
"containing it are trainable. Hence when we need to partially freeze/unfreeze\n",
@@ -556,19 +719,92 @@
},
"outputs": [],
"source": [
- "def unfreeze_model(model):\n",
- " # We unfreeze the top 20 layers while leaving BatchNorm layers frozen\n",
- " for layer in model.layers[-20:]:\n",
+ "\n",
+ "def unfreeze_model(\n",
+ " model,\n",
+ " layers_to_unfreeze=20,\n",
+ " learning_rate=1e-5,\n",
+ " loss=\"categorical_crossentropy\",\n",
+ " metrics=None,\n",
+ "):\n",
+ " \"\"\"Unfreeze part of `model` and recompile it for fine-tuning.\n",
+ "\n",
+ " Args:\n",
+ " model: A `keras.Model` instance to unfreeze in place.\n",
+ " layers_to_unfreeze: Either an `int` giving the number of layers,\n",
+ " counted from the end of the base model's layers, to unfreeze, or a `str`\n",
+ " substring to match against layer names -- the first matching\n",
+ " layer and every layer after it (in the base model's layers order) are\n",
+ " unfrozen. Use a string like `\"block7\"` to respect EfficientNet's\n",
+ " residual block boundaries instead of an arbitrary layer count.\n",
+ " Defaults to `20`.\n",
+ " learning_rate: Learning rate for the fine-tuning `Adam` optimizer.\n",
+ " Defaults to `1e-5`.\n",
+ " loss: Loss function passed to `model.compile()`. Defaults to\n",
+ " `\"categorical_crossentropy\"`.\n",
+ " metrics: List of metrics passed to `model.compile()`. Defaults to\n",
+ " `[\"accuracy\"]`.\n",
+ "\n",
+ " Returns:\n",
+ " `model`, with the selected layers unfrozen (except\n",
+ " `BatchNormalization` layers, which are always kept frozen) and\n",
+ " recompiled with the new optimizer/loss/metrics.\n",
+ " \"\"\"\n",
+ " if metrics is None:\n",
+ " metrics = [\"accuracy\"]\n",
+ "\n",
+ " # Access the nested EfficientNet base model by finding the first layer\n",
+ " # with 'efficientnet' in its name (case-insensitive)\n",
+ " base_model = None\n",
+ " for layer in model.layers:\n",
+ " if \"efficientnet\" in layer.name.lower():\n",
+ " base_model = layer\n",
+ " break\n",
+ " if base_model is None:\n",
+ " raise ValueError(\n",
+ " \"Could not find EfficientNet base model in the model. \"\n",
+ " \"Expected a layer with 'efficientnet' in its name.\"\n",
+ " )\n",
+ " base_model.trainable = True\n",
+ "\n",
+ " # First, freeze all layers in the base model\n",
+ " for layer in base_model.layers:\n",
+ " layer.trainable = False\n",
+ "\n",
+ " if isinstance(layers_to_unfreeze, str):\n",
+ " unfreeze_from = None\n",
+ " for i, layer in enumerate(base_model.layers):\n",
+ " if layers_to_unfreeze in layer.name:\n",
+ " unfreeze_from = i\n",
+ " break\n",
+ " if unfreeze_from is None:\n",
+ " raise ValueError(f\"No layer name contains {layers_to_unfreeze!r}.\")\n",
+ " layers_to_process = base_model.layers[unfreeze_from:]\n",
+ " elif isinstance(layers_to_unfreeze, int):\n",
+ " if layers_to_unfreeze <= 0:\n",
+ " raise ValueError(\n",
+ " f\"layers_to_unfreeze must be > 0, got {layers_to_unfreeze}\"\n",
+ " )\n",
+ " n_layers_to_unfreeze = min(layers_to_unfreeze, len(base_model.layers))\n",
+ " layers_to_process = base_model.layers[-n_layers_to_unfreeze:]\n",
+ " else:\n",
+ " raise TypeError(\n",
+ " \"layers_to_unfreeze must be an int or str, received: \"\n",
+ " f\"{type(layers_to_unfreeze)}\"\n",
+ " )\n",
+ "\n",
+ " # We keep BatchNorm layers frozen -- see \"Tips for fine-tuning\n",
+ " # EfficientNet\" in the tutorial for why.\n",
+ " for layer in layers_to_process:\n",
" if not isinstance(layer, layers.BatchNormalization):\n",
" layer.trainable = True\n",
"\n",
- " optimizer = keras.optimizers.Adam(learning_rate=1e-5)\n",
- " model.compile(\n",
- " optimizer=optimizer, loss=\"categorical_crossentropy\", metrics=[\"accuracy\"]\n",
- " )\n",
+ " optimizer = keras.optimizers.Adam(learning_rate=learning_rate)\n",
+ " model.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n",
+ " return model\n",
"\n",
"\n",
- "unfreeze_model(model)\n",
+ "model = unfreeze_model(model)\n",
"\n",
"epochs = 4 # @param {type: \"slider\", min:4, max:10}\n",
"hist = model.fit(ds_train, epochs=epochs, validation_data=ds_test)\n",
@@ -581,43 +817,45 @@
"colab_type": "text"
},
"source": [
- "### Tips for fine tuning EfficientNet\n",
+ "### Tips for fine-tuning EfficientNet\n",
"\n",
- "On unfreezing layers:\n",
+ "**On unfreezing layers:**\n",
"\n",
"- The `BatchNormalization` layers need to be kept frozen\n",
"([more details](https://keras.io/guides/transfer_learning/)).\n",
"If they are also turned to trainable, the\n",
"first epoch after unfreezing will significantly reduce accuracy.\n",
- "- In some cases it may be beneficial to open up only a portion of layers instead of\n",
- "unfreezing all. This will make fine tuning much faster when going to larger models like\n",
+ "- In some cases it may be beneficial to unfreeze only a portion of layers instead of\n",
+ "unfreezing all. This will make fine-tuning much faster when going to larger models like\n",
"B7.\n",
"- Each block needs to be all turned on or off. This is because the architecture includes\n",
"a shortcut from the first layer to the last layer for each block. Not respecting blocks\n",
"also significantly harms the final performance.\n",
"\n",
- "Some other tips for utilizing EfficientNet:\n",
+ "**Some other tips for utilizing EfficientNet:**\n",
"\n",
"- Larger variants of EfficientNet do not guarantee improved performance, especially for\n",
- "tasks with less data or fewer classes. In such a case, the larger variant of EfficientNet\n",
+ "tasks with less data or fewer classes. In such a case, the larger the variant of EfficientNet\n",
"chosen, the harder it is to tune hyperparameters.\n",
"- EMA (Exponential Moving Average) is very helpful in training EfficientNet from scratch,\n",
"but not so much for transfer learning.\n",
"- Do not use the RMSprop setup as in the original paper for transfer learning. The\n",
"momentum and learning rate are too high for transfer learning. It will easily corrupt the\n",
- "pretrained weight and blow up the loss. A quick check is to see if loss (as categorical\n",
+ "pretrained weights and blow up the loss. A quick check is to see if loss (as categorical\n",
"cross entropy) is getting significantly larger than log(NUM_CLASSES) after the same\n",
"epoch. If so, the initial learning rate/momentum is too high.\n",
- "- Smaller batch size benefit validation accuracy, possibly due to effectively providing\n",
+ "- Smaller batch sizes benefit validation accuracy, possibly due to effectively providing\n",
"regularization."
]
},
{
"cell_type": "markdown",
- "metadata": {},
+ "metadata": {
+ "colab_type": "text"
+ },
"source": [
"## Relevant Chapters from Deep Learning with Python\n",
- "- [Chapter 8: Image classification](https://deeplearningwithpython.io/chapters/chapter08_image-classification)\n"
+ "- [Chapter 8: Image classification](https://deeplearningwithpython.io/chapters/chapter08_image-classification)"
]
}
],
@@ -650,4 +888,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
-}
+}
\ No newline at end of file
diff --git a/examples/vision/md/image_classification_efficientnet_fine_tuning.md b/examples/vision/md/image_classification_efficientnet_fine_tuning.md
index c325b3175e..ec8599c4c0 100644
--- a/examples/vision/md/image_classification_efficientnet_fine_tuning.md
+++ b/examples/vision/md/image_classification_efficientnet_fine_tuning.md
@@ -2,7 +2,7 @@
**Author:** [Yixing Fu](https://github.com/yixingfu)
**Date created:** 2020/06/30
-**Last modified:** 2023/07/10
+**Last modified:** 2026/07/13
**Description:** Use EfficientNet with weights pre-trained on imagenet for Stanford Dogs classification.
@@ -27,12 +27,6 @@ heuristics (compound-scaling, details see
efficiency-oriented base model (B0) to surpass models at every scale, while avoiding
extensive grid-search of hyperparameters.
-A summary of the latest updates on the model is available at
-[here](https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet), where various
-augmentation schemes and semi-supervised learning approaches are applied to further
-improve the imagenet performance of the models. These extensions of the model can be used
-by updating weights without changing model architecture.
-
---
## B0 to B7 variants of EfficientNet
@@ -60,7 +54,7 @@ As a result, the depth, width and resolution of each variant of the EfficientNet
are hand-picked and proven to produce good results, though they may be significantly
off from the compound scaling formula.
Therefore, the keras implementation (detailed below) only provide these 8 models, B0 to B7,
-instead of allowing arbitray choice of width / depth / resolution parameters.
+instead of allowing arbitrary choice of width / depth / resolution parameters.
---
## Keras implementation of EfficientNet
@@ -69,7 +63,7 @@ An implementation of EfficientNet B0 to B7 has been shipped with Keras since v2.
use EfficientNetB0 for classifying 1000 classes of images from ImageNet, run:
```python
-from tensorflow.keras.applications import EfficientNetB0
+from keras.applications import EfficientNetB0
model = EfficientNetB0(weights='imagenet')
```
@@ -127,10 +121,13 @@ As an end-to-end example, we will show using pre-trained EfficientNetB0 on
```python
+import tarfile
+import urllib.request
+from pathlib import Path
import numpy as np
-import tensorflow_datasets as tfds
-import tensorflow as tf # For tf.data
import matplotlib.pyplot as plt
+from PIL import Image
+import scipy.io
import keras
from keras import layers
from keras.applications import EfficientNetB0
@@ -143,42 +140,126 @@ BATCH_SIZE = 64
### Loading data
-Here we load data from [tensorflow_datasets](https://www.tensorflow.org/datasets)
-(hereafter TFDS).
-Stanford Dogs dataset is provided in
-TFDS as [stanford_dogs](https://www.tensorflow.org/datasets/catalog/stanford_dogs).
-It features 20,580 images that belong to 120 classes of dog breeds
+We download the Stanford Dogs dataset directly from Stanford's servers using the
+built-in `urllib` and `tarfile` modules.
+The dataset contains 20,580 images belonging to 120 classes of dog breeds
(12,000 for training and 8,580 for testing).
-By simply changing `dataset_name` below, you may also try this notebook for
-other datasets in TFDS such as
-[cifar10](https://www.tensorflow.org/datasets/catalog/cifar10),
-[cifar100](https://www.tensorflow.org/datasets/catalog/cifar100),
-[food101](https://www.tensorflow.org/datasets/catalog/food101),
-etc. When the images are much smaller than the size of EfficientNet input,
-we can simply upsample the input images. It has been shown in
-[Tan and Le, 2019](https://arxiv.org/abs/1905.11946) that transfer learning
-result is better for increased resolution even if input images remain small.
+The dataset is downloaded, extracted, and loaded into lists of NumPy arrays.
+Images have variable dimensions and will be resized to a uniform size in the data pipeline.
+
+**Note:** This direct download approach eliminates dependency conflicts that can
+occur with `tensorflow_datasets` in some environments (particularly Google Colab
+with protobuf version incompatibilities).
```python
-dataset_name = "stanford_dogs"
-(ds_train, ds_test), ds_info = tfds.load(
- dataset_name, split=["train", "test"], with_info=True, as_supervised=True
+dataset_url = "http://vision.stanford.edu/aditya86/ImageNetDogs/images.tar"
+lists_url = "http://vision.stanford.edu/aditya86/ImageNetDogs/lists.tar"
+data_dir = Path("./stanford_dogs_data")
+data_dir.mkdir(exist_ok=True)
+
+
+def download_and_extract(url, extract_to):
+ filename = url.split("/")[-1]
+ filepath = data_dir / filename
+ if not filepath.exists():
+ print(f"Downloading {filename}...")
+ urllib.request.urlretrieve(url, filepath)
+ print(f"Extracting {filename}...")
+ with tarfile.open(filepath, "r") as tar:
+ tar.extractall(extract_to, filter="data")
+ return extract_to
+
+
+# Download dataset
+images_dir = download_and_extract(dataset_url, data_dir)
+lists_dir = download_and_extract(lists_url, data_dir)
+
+# Parse train/test splits
+def load_file_list(filepath):
+ mat = scipy.io.loadmat(filepath)
+ return [item[0][0] for item in mat["file_list"]]
+
+
+train_files = load_file_list(data_dir / "train_list.mat")
+test_files = load_file_list(data_dir / "test_list.mat")
+
+# Build class name mapping
+all_files = train_files + test_files
+class_names = sorted(set([f.split("/")[0] for f in all_files]))
+class_to_idx = {name: idx for idx, name in enumerate(class_names)}
+NUM_CLASSES = len(class_names)
+
+print(
+ f"Found {NUM_CLASSES} classes, {len(train_files)} training images, {len(test_files)} test images"
)
-NUM_CLASSES = ds_info.features["label"].num_classes
+
+# Prepare image paths and labels (lazy loading - no images loaded into memory yet)
+def prepare_paths_and_labels(file_list, base_dir):
+ image_paths, labels = [], []
+ for file_path in file_list:
+ class_name = file_path.split("/")[0]
+ img_path = base_dir / "Images" / file_path
+ if img_path.exists():
+ image_paths.append(str(img_path))
+ labels.append(class_to_idx[class_name])
+ return image_paths, np.array(labels)
+
+
+print("Preparing dataset paths...")
+train_image_paths, train_labels = prepare_paths_and_labels(train_files, data_dir)
+test_image_paths, test_labels = prepare_paths_and_labels(test_files, data_dir)
+print(f"Found {len(train_image_paths)} train and {len(test_image_paths)} test images")
```
-When the dataset include images with various size, we need to resize them into a
-shared size. The Stanford Dogs dataset includes only images at least 200x200
-pixels in size. Here we resize the images to the input size needed for EfficientNet.
+Each image can have a different shape, so we resize them to a shared input size
+for EfficientNet. In Keras 3, we do this in a backend-agnostic `PyDataset` pipeline
+using `keras.ops.image.resize`, which works across TensorFlow, JAX, and PyTorch backends,
+rather than the TensorFlow-specific `tf.data` mapping steps.
+
+Images are loaded lazily from disk in `__getitem__` to avoid loading all 20,580 images
+into memory at once, which would consume several gigabytes of RAM and cause OOM issues.
```python
-size = (IMG_SIZE, IMG_SIZE)
-ds_train = ds_train.map(lambda image, label: (tf.image.resize(image, size), label))
-ds_test = ds_test.map(lambda image, label: (tf.image.resize(image, size), label))
+class ResizeOnlyDataset(keras.utils.PyDataset):
+ def __init__(self, image_paths, labels, img_size, batch_size=1, **kwargs):
+ super().__init__(**kwargs)
+ self.image_paths = image_paths
+ self.labels = labels
+ self.img_size = img_size
+ self.batch_size = batch_size
+ self.indices = np.arange(len(labels))
+
+ def __len__(self):
+ return int(np.ceil(len(self.labels) / self.batch_size))
+
+ def __getitem__(self, idx):
+ batch_indices = self.indices[
+ idx * self.batch_size : (idx + 1) * self.batch_size
+ ]
+ batch_images = []
+ for i in batch_indices:
+ img = Image.open(self.image_paths[i]).convert("RGB")
+ img_array = np.array(img, dtype="float32")
+ img_resized = keras.ops.image.resize(
+ img_array, (self.img_size, self.img_size)
+ )
+ batch_images.append(np.array(img_resized))
+ batch_images = np.stack(batch_images)
+ batch_labels = self.labels[batch_indices]
+ if self.batch_size == 1:
+ return batch_images[0], batch_labels[0]
+ return batch_images, batch_labels
+
+
+# Preview stream with resized images for visualization below
+preview_train = ResizeOnlyDataset(
+ train_image_paths, train_labels, IMG_SIZE, batch_size=1
+)
+preview_test = ResizeOnlyDataset(test_image_paths, test_labels, IMG_SIZE, batch_size=1)
```
### Visualizing the data
@@ -189,14 +270,15 @@ The following code shows the first 9 images with their labels.
```python
def format_label(label):
- string_label = label_info.int2str(label)
- return string_label.split("-")[1]
+ class_name = class_names[int(label)]
+ return class_name.split("-")[1] # Extract breed name from "n02085620-Chihuahua"
-label_info = ds_info.features["label"]
-for i, (image, label) in enumerate(ds_train.take(9)):
+for i, (image, label) in enumerate(preview_train):
+ if i >= 9:
+ break
ax = plt.subplot(3, 3, i + 1)
- plt.imshow(image.numpy().astype("uint8"))
+ plt.imshow(np.array(image).astype("uint8"))
plt.title("{}".format(format_label(label)))
plt.axis("off")
@@ -210,7 +292,8 @@ for i, (image, label) in enumerate(ds_train.take(9)):
### Data augmentation
-We can use the preprocessing layers APIs for image augmentation.
+We can use Keras preprocessing layers for image augmentation.
+These layers are backend-agnostic and can be used during both training and inference.
```python
@@ -229,22 +312,22 @@ def img_augmentation(images):
```
-This `Sequential` model object can be used both as a part of
-the model we later build, and as a function to preprocess
-data before feeding into the model. Using them as function makes
-it easy to visualize the augmented images. Here we plot 9 examples
-of augmentation result of a given figure.
+The `img_augmentation` function can be used both as a part of the model
+we later build, and as a standalone function to preprocess data before feeding
+into the model. Using it as a function makes it easy to visualize the augmentation
+results. Here we plot 9 examples of augmentation applied to a single image.
```python
-for image, label in ds_train.take(1):
- for i in range(9):
- ax = plt.subplot(3, 3, i + 1)
- aug_img = img_augmentation(np.expand_dims(image.numpy(), axis=0))
- aug_img = np.array(aug_img)
- plt.imshow(aug_img[0].astype("uint8"))
- plt.title("{}".format(format_label(label)))
- plt.axis("off")
+first_image, first_label = preview_train[0]
+
+for i in range(9):
+ ax = plt.subplot(3, 3, i + 1)
+ aug_img = img_augmentation(np.expand_dims(np.array(first_image), axis=0))
+ aug_img = np.array(aug_img)
+ plt.imshow(aug_img[0].astype("uint8"))
+ plt.title("{}".format(format_label(first_label)))
+ plt.axis("off")
```
@@ -257,45 +340,104 @@ for image, label in ds_train.take(1):
### Prepare inputs
Once we verify the input data and augmentation are working correctly,
-we prepare dataset for training. The input data are resized to uniform
-`IMG_SIZE`. The labels are put into one-hot
-(a.k.a. categorical) encoding. The dataset is batched.
-
-Note: `prefetch` and `AUTOTUNE` may in some situation improve
-performance, but depends on environment and the specific dataset used.
-See this [guide](https://www.tensorflow.org/guide/data_performance)
-for more information on data pipeline performance.
-
-
-```python
+we prepare backend-agnostic datasets for training.
-# One-hot / categorical encoding
-def input_preprocess_train(image, label):
- image = img_augmentation(image)
- label = tf.one_hot(label, NUM_CLASSES)
- return image, label
+The input images are resized to uniform `IMG_SIZE`, labels are converted to one-hot
+(categorical) encoding, and batches are produced by `keras.utils.PyDataset`.
+Compared to the original `tf.data` version, this Keras 3 setup is backend-agnostic
+and works seamlessly across TensorFlow, JAX, and PyTorch backends.
-def input_preprocess_test(image, label):
- label = tf.one_hot(label, NUM_CLASSES)
- return image, label
+```python
+class StanfordDogsDataset(keras.utils.PyDataset):
+ def __init__(
+ self,
+ image_paths,
+ labels,
+ num_classes,
+ img_size,
+ batch_size,
+ augment=False,
+ shuffle=False,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self.image_paths = image_paths
+ self.labels = labels
+ self.num_classes = num_classes
+ self.img_size = img_size
+ self.batch_size = batch_size
+ self.augment = augment
+ self.shuffle = shuffle
+ self.indices = np.arange(len(labels))
+ self.on_epoch_end()
+
+ def __len__(self):
+ # Match previous drop_remainder=True behavior.
+ return len(self.indices) // self.batch_size
+
+ def on_epoch_end(self):
+ if self.shuffle:
+ np.random.shuffle(self.indices)
+
+ def __getitem__(self, idx):
+ batch_indices = self.indices[
+ idx * self.batch_size : (idx + 1) * self.batch_size
+ ]
+
+ # Load images lazily from disk
+ batch_images = []
+ for i in batch_indices:
+ img = Image.open(self.image_paths[i]).convert("RGB")
+ img_array = np.array(img, dtype="float32")
+ img_resized = keras.ops.image.resize(
+ img_array, (self.img_size, self.img_size)
+ )
+ batch_images.append(np.array(img_resized))
+ batch_images = np.stack(batch_images)
+
+ if self.augment:
+ batch_images = np.array(img_augmentation(batch_images))
+
+ batch_labels = np.array(
+ keras.ops.one_hot(self.labels[batch_indices], self.num_classes)
+ )
+
+ return batch_images, batch_labels
+
+
+ds_train = StanfordDogsDataset(
+ train_image_paths,
+ train_labels,
+ num_classes=NUM_CLASSES,
+ img_size=IMG_SIZE,
+ batch_size=BATCH_SIZE,
+ augment=True,
+ shuffle=True,
+ workers=2,
+ use_multiprocessing=False,
+)
-ds_train = ds_train.map(input_preprocess_train, num_parallel_calls=tf.data.AUTOTUNE)
-ds_train = ds_train.batch(batch_size=BATCH_SIZE, drop_remainder=True)
-ds_train = ds_train.prefetch(tf.data.AUTOTUNE)
-
-ds_test = ds_test.map(input_preprocess_test, num_parallel_calls=tf.data.AUTOTUNE)
-ds_test = ds_test.batch(batch_size=BATCH_SIZE, drop_remainder=True)
-
+ds_test = StanfordDogsDataset(
+ test_image_paths,
+ test_labels,
+ num_classes=NUM_CLASSES,
+ img_size=IMG_SIZE,
+ batch_size=BATCH_SIZE,
+ augment=False,
+ shuffle=False,
+)
```
---
## Training a model from scratch
-We build an EfficientNetB0 with 120 output classes, that is initialized from scratch:
+We build an EfficientNetB0 with 120 output classes, initialized from scratch
+(no pretrained weights).
-Note: the accuracy will increase very slowly and may overfit.
+**Note:** Training from scratch typically shows slower convergence and may overfit
+on smaller datasets like Stanford Dogs.
```python
@@ -318,9 +460,6 @@ hist = model.fit(ds_train, epochs=epochs, validation_data=ds_test)
Model: "efficientnetb0"
-
-
-
┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓
┃ Layer (type) ┃ Output Shape ┃ Param # ┃ Connected to ┃
┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩
@@ -1045,21 +1184,12 @@ hist = model.fit(ds_train, epochs=epochs, validation_data=ds_test)
└─────────────────────┴───────────────────┴─────────┴──────────────────────┘
-
-
-
Total params: 4,203,291 (16.03 MB)
-
-
-
Trainable params: 4,161,268 (15.87 MB)
-
-
-
Non-trainable params: 42,023 (164.16 KB)
@@ -1155,13 +1285,13 @@ Epoch 40/40
```
-Training the model is relatively fast. This might make it sounds easy to simply train EfficientNet on any
-dataset wanted from scratch. However, training EfficientNet on smaller datasets,
+Training the model is relatively fast (a few minutes per epoch on modern hardware).
+However, training EfficientNet on smaller datasets,
especially those with lower resolution like CIFAR-100, faces the significant challenge of
overfitting.
-Hence training from scratch requires very careful choice of hyperparameters and is
-difficult to find suitable regularization. It would also be much more demanding in resources.
+Training from scratch requires very careful choice of hyperparameters and
+suitable regularization. It is also much more demanding in computational resources.
Plotting the training and validation accuracy
makes it clear that validation accuracy stagnates at a low value.
@@ -1190,10 +1320,11 @@ plot_hist(hist)
---
-## Transfer learning from pre-trained weights
+## Transfer learning from pretrained weights
-Here we initialize the model with pre-trained ImageNet weights,
-and we fine-tune it on our own dataset.
+Here we initialize the model with pretrained ImageNet weights
+and fine-tune it on our own dataset. This is the recommended approach
+for most applications.
```python
@@ -1223,15 +1354,15 @@ def build_model(num_classes):
```
-The first step to transfer learning is to freeze all layers and train only the top
+The first step in transfer learning is to freeze all base layers and train only the top
layers. For this step, a relatively large learning rate (1e-2) can be used.
-Note that validation accuracy and loss will usually be better than training
+
+**Note:** Validation accuracy and loss will usually be better than training
accuracy and loss. This is because the regularization is strong, which only
suppresses training-time metrics.
-Note that the convergence may take up to 50 epochs depending on choice of learning rate.
-If image augmentation layers were not
-applied, the validation accuracy may only reach ~60%.
+The convergence may take up to 50 epochs depending on the choice of learning rate.
+If image augmentation layers were not applied, the validation accuracy may only reach ~60%.
```python
@@ -1302,14 +1433,18 @@ Epoch 25/25
-The second step is to unfreeze a number of layers and fit the model using smaller
-learning rate. In this example we show unfreezing all layers, but depending on
-specific dataset it may be desireble to only unfreeze a fraction of all layers.
+The second step is to unfreeze a number of layers and fine-tune the model using a smaller
+learning rate. In this example we unfreeze the last 20 layers, but depending on the
+specific dataset it may be desirable to only unfreeze a fraction of all layers.
-When the feature extraction with
-pretrained model works good enough, this step would give a very limited gain on
-validation accuracy. In our case we only see a small improvement,
-as ImageNet pretraining already exposed the model to a good amount of dogs.
+**Advanced usage:** The `unfreeze_model()` function also supports unfreezing by block name
+(e.g., `unfreeze_model(model, layers_to_unfreeze="block7")`) to respect EfficientNet's
+residual block boundaries. See the "Tips for fine-tuning EfficientNet" section below
+for why this matters.
+
+When feature extraction with the pretrained model works well enough, this step provides
+only a limited gain in validation accuracy. In our case we only see a small improvement,
+as ImageNet pretraining already exposed the model to a good amount of dog images.
On the other hand, when we use pretrained weights on a dataset that is more different
from ImageNet, this fine-tuning step can be crucial as the feature extractor also
@@ -1317,7 +1452,7 @@ needs to be adjusted by a considerable amount. Such a situation can be demonstra
if choosing CIFAR-100 dataset instead, where fine-tuning boosts validation accuracy
by about 10% to pass 80% on `EfficientNetB0`.
-A side note on freezing/unfreezing models: setting `trainable` of a `Model` will
+**Note on freezing/unfreezing models:** Setting `trainable` of a `Model` will
simultaneously set all layers belonging to the `Model` to the same `trainable`
attribute. Each layer is trainable only if both the layer itself and the model
containing it are trainable. Hence when we need to partially freeze/unfreeze
@@ -1327,19 +1462,91 @@ to `True`.
```python
-def unfreeze_model(model):
- # We unfreeze the top 20 layers while leaving BatchNorm layers frozen
- for layer in model.layers[-20:]:
+def unfreeze_model(
+ model,
+ layers_to_unfreeze=20,
+ learning_rate=1e-5,
+ loss="categorical_crossentropy",
+ metrics=None,
+):
+ """Unfreeze part of `model` and recompile it for fine-tuning.
+
+ Args:
+ model: A `keras.Model` instance to unfreeze in place.
+ layers_to_unfreeze: Either an `int` giving the number of layers,
+ counted from the end of the base model's layers, to unfreeze, or a `str`
+ substring to match against layer names -- the first matching
+ layer and every layer after it (in the base model's layers order) are
+ unfrozen. Use a string like `"block7"` to respect EfficientNet's
+ residual block boundaries instead of an arbitrary layer count.
+ Defaults to `20`.
+ learning_rate: Learning rate for the fine-tuning `Adam` optimizer.
+ Defaults to `1e-5`.
+ loss: Loss function passed to `model.compile()`. Defaults to
+ `"categorical_crossentropy"`.
+ metrics: List of metrics passed to `model.compile()`. Defaults to
+ `["accuracy"]`.
+
+ Returns:
+ `model`, with the selected layers unfrozen (except
+ `BatchNormalization` layers, which are always kept frozen) and
+ recompiled with the new optimizer/loss/metrics.
+ """
+ if metrics is None:
+ metrics = ["accuracy"]
+
+ # Access the nested EfficientNet base model by finding the first layer
+ # with 'efficientnet' in its name (case-insensitive)
+ base_model = None
+ for layer in model.layers:
+ if "efficientnet" in layer.name.lower():
+ base_model = layer
+ break
+ if base_model is None:
+ raise ValueError(
+ "Could not find EfficientNet base model in the model. "
+ "Expected a layer with 'efficientnet' in its name."
+ )
+ base_model.trainable = True
+
+ # First, freeze all layers in the base model
+ for layer in base_model.layers:
+ layer.trainable = False
+
+ if isinstance(layers_to_unfreeze, str):
+ unfreeze_from = None
+ for i, layer in enumerate(base_model.layers):
+ if layers_to_unfreeze in layer.name:
+ unfreeze_from = i
+ break
+ if unfreeze_from is None:
+ raise ValueError(f"No layer name contains {layers_to_unfreeze!r}.")
+ layers_to_process = base_model.layers[unfreeze_from:]
+ elif isinstance(layers_to_unfreeze, int):
+ if layers_to_unfreeze <= 0:
+ raise ValueError(
+ f"layers_to_unfreeze must be > 0, got {layers_to_unfreeze}"
+ )
+ n_layers_to_unfreeze = min(layers_to_unfreeze, len(base_model.layers))
+ layers_to_process = base_model.layers[-n_layers_to_unfreeze:]
+ else:
+ raise TypeError(
+ "layers_to_unfreeze must be an int or str, received: "
+ f"{type(layers_to_unfreeze)}"
+ )
+
+ # We keep BatchNorm layers frozen -- see "Tips for fine-tuning
+ # EfficientNet" in the tutorial for why.
+ for layer in layers_to_process:
if not isinstance(layer, layers.BatchNormalization):
layer.trainable = True
- optimizer = keras.optimizers.Adam(learning_rate=1e-5)
- model.compile(
- optimizer=optimizer, loss="categorical_crossentropy", metrics=["accuracy"]
- )
+ optimizer = keras.optimizers.Adam(learning_rate=learning_rate)
+ model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
+ return model
-unfreeze_model(model)
+model = unfreeze_model(model)
epochs = 4 # @param {type: "slider", min:4, max:10}
hist = model.fit(ds_train, epochs=epochs, validation_data=ds_test)
@@ -1358,40 +1565,41 @@ Epoch 4/4
187/187 ━━━━━━━━━━━━━━━━━━━━ 79s 419ms/step - accuracy: 0.6625 - loss: 1.1775 - val_accuracy: 0.7701 - val_loss: 0.8284
```
+
-### Tips for fine tuning EfficientNet
+### Tips for fine-tuning EfficientNet
-On unfreezing layers:
+**On unfreezing layers:**
- The `BatchNormalization` layers need to be kept frozen
([more details](https://keras.io/guides/transfer_learning/)).
If they are also turned to trainable, the
first epoch after unfreezing will significantly reduce accuracy.
-- In some cases it may be beneficial to open up only a portion of layers instead of
-unfreezing all. This will make fine tuning much faster when going to larger models like
+- In some cases it may be beneficial to unfreeze only a portion of layers instead of
+unfreezing all. This will make fine-tuning much faster when going to larger models like
B7.
- Each block needs to be all turned on or off. This is because the architecture includes
a shortcut from the first layer to the last layer for each block. Not respecting blocks
also significantly harms the final performance.
-Some other tips for utilizing EfficientNet:
+**Some other tips for utilizing EfficientNet:**
- Larger variants of EfficientNet do not guarantee improved performance, especially for
-tasks with less data or fewer classes. In such a case, the larger variant of EfficientNet
+tasks with less data or fewer classes. In such a case, the larger the variant of EfficientNet
chosen, the harder it is to tune hyperparameters.
- EMA (Exponential Moving Average) is very helpful in training EfficientNet from scratch,
but not so much for transfer learning.
- Do not use the RMSprop setup as in the original paper for transfer learning. The
momentum and learning rate are too high for transfer learning. It will easily corrupt the
-pretrained weight and blow up the loss. A quick check is to see if loss (as categorical
+pretrained weights and blow up the loss. A quick check is to see if loss (as categorical
cross entropy) is getting significantly larger than log(NUM_CLASSES) after the same
epoch. If so, the initial learning rate/momentum is too high.
-- Smaller batch size benefit validation accuracy, possibly due to effectively providing
+- Smaller batch sizes benefit validation accuracy, possibly due to effectively providing
regularization.
---