88import os
99import tempfile
1010import zipfile
11+ from collections .abc import Sized
1112
1213from pathlib import Path
1314from typing import Any , Optional , Tuple
1415
1516import torch
1617from torch .nn .modules import Module
1718from torch .utils ._pytree import tree_flatten
18- from torch .utils .data import DataLoader
19+ from torch .utils .data import DataLoader , Dataset , Subset
1920from torchvision import datasets , transforms # type: ignore[import-untyped]
2021
2122
@@ -109,32 +110,49 @@ def __init__(
109110 batch_size : int ,
110111 validation_dataset_path : str ,
111112 eval_dtype : Optional [torch .dtype ] = None ,
113+ max_samples : Optional [int ] = None ,
114+ validation_dataset : Optional [Dataset [Any ]] = None ,
112115 ) -> None :
113116 self ._model_name = model_name
114117 self ._eval_model = eval_model
115118 self ._batch_size = batch_size
116119 self ._validation_set_path = validation_dataset_path
117120 self ._eval_dtype = eval_dtype
121+ if max_samples is not None and max_samples <= 0 :
122+ raise ValueError ("max_samples must be greater than zero" )
123+ self ._max_samples = max_samples
124+ self ._validation_dataset = validation_dataset
118125
119126 def evaluate (self ) -> dict [str , Any ]:
120127 metrics : dict [str , Any ] = {}
121128
122- dataset = ImageNetEvaluator .load_imagenet_folder (self ._validation_set_path )
129+ dataset : Dataset [Any ]
130+ if self ._validation_dataset is None :
131+ dataset = ImageNetEvaluator .load_imagenet_folder (self ._validation_set_path )
132+ else :
133+ dataset = self ._validation_dataset
134+ if not isinstance (dataset , Sized ):
135+ raise TypeError ("validation dataset must define __len__" )
136+ dataset_size = len (dataset )
137+ if self ._max_samples is not None :
138+ dataset_size = min (self ._max_samples , dataset_size )
139+ dataset = Subset (dataset , range (dataset_size ))
123140 logger .debug (
124141 "Starting ImageNet evaluation for model '%s' on dataset '%s' with %d samples." ,
125142 self ._model_name ,
126143 self ._validation_set_path ,
127- len ( dataset ) ,
144+ dataset_size ,
128145 )
129146
130147 top1 , top5 = self ._evaluate_topk (dataset , topk = 5 )
148+ metrics ["num_samples" ] = dataset_size
131149 metrics ["accuracy" ] = {"top-1" : top1 , "top-5" : top5 }
132150
133151 return metrics
134152
135153 def _evaluate_topk (
136154 self ,
137- dataset : datasets . ImageFolder ,
155+ dataset : Dataset [ Any ] ,
138156 topk : int = 5 ,
139157 log_every : int = 50 ,
140158 ) -> Tuple [float , float ]:
@@ -170,6 +188,11 @@ def _evaluate_topk(
170188 logger .debug (
171189 "Model eval() unsupported and torchao allow_exported_model_train_eval not available; proceeding."
172190 )
191+ if not isinstance (dataset , Sized ):
192+ raise TypeError ("validation dataset must define __len__" )
193+ dataset_size = len (dataset )
194+ if dataset_size == 0 :
195+ raise ValueError ("validation dataset must contain at least one sample" )
173196 loaded_dataset = DataLoader (dataset , batch_size = self ._batch_size , shuffle = False )
174197 top1_correct = 0
175198 topk_correct = 0
@@ -187,17 +210,17 @@ def _evaluate_topk(
187210 topk_correct += (topk_indices == target_view ).sum ().item ()
188211 batch_sz = image .size (0 )
189212 total += batch_sz
190- if (i + 1 ) % log_every == 0 or total == len ( dataset ) :
213+ if (i + 1 ) % log_every == 0 or total == dataset_size :
191214 logger .info (
192215 "Eval progress: %d / %d top1=%.4f top%d=%.4f" ,
193216 total ,
194- len ( dataset ) ,
217+ dataset_size ,
195218 top1_correct / total ,
196219 topk ,
197220 topk_correct / total ,
198221 )
199- top1_accuracy = top1_correct / len ( dataset )
200- topk_accuracy = topk_correct / len ( dataset )
222+ top1_accuracy = top1_correct / dataset_size
223+ topk_accuracy = topk_correct / dataset_size
201224
202225 return top1_accuracy , topk_accuracy
203226
0 commit comments