diff --git a/src/machinevisiontoolbox/BagOfWords.py b/src/machinevisiontoolbox/BagOfWords.py index 65b85885..33231f3e 100644 --- a/src/machinevisiontoolbox/BagOfWords.py +++ b/src/machinevisiontoolbox/BagOfWords.py @@ -58,6 +58,13 @@ def __init__( of image features is assigned labels from the ``.centroids`` any with a label greater that ``.nstopwords`` is a stop word and can be discarded. + .. note:: Removing stop words can leave a remaining word that never + occurs in any image, giving it an inverse document frequency (idf) + of ``inf`` (``log(N / 0)``). This is harmless -- every place idf is + used to compute a word-frequency vector, the corresponding word + count is also always zero there, so the ``0 * inf`` result is + ``nan`` and gets replaced with 0 before the vector is returned. + :reference: - Video Google: a text retrieval approach to object matching in videos J.Sivic and A.Zisserman, @@ -139,7 +146,12 @@ def __init__( # total number of occurrences of word i # multiple occurrences in the one image count only as one ni = (W > 0).sum(axis=1) - idf = np.log(N / ni) + # a word that never occurs in any image (ni=0, possible after + # stopword removal) gives idf=inf here; harmless since nid is + # always 0 at that same position below, so 0*inf->nan, already + # suppressed by the invalid="ignore" there. + with np.errstate(divide="ignore"): + idf = np.log(N / ni) M = [] diff --git a/tests/test_bag_of_words.py b/tests/test_bag_of_words.py index e8ab1f13..bd34e619 100644 --- a/tests/test_bag_of_words.py +++ b/tests/test_bag_of_words.py @@ -4,6 +4,7 @@ """ import unittest +import warnings from machinevisiontoolbox import FileCollection, BagOfWords import numpy as np @@ -26,7 +27,15 @@ def test_retrieve(self): bag = BagOfWords(features, 2_000, seed=0) # self.assertEqual(len(bag), 42_213) - bag = BagOfWords(features, 2_000, nstopwords=50, seed=0) + # Regression test: nstopwords>0 can leave a word with zero + # occurrences across every remaining image (ni=0), which used to + # raise "RuntimeWarning: divide by zero" from idf = np.log(N/ni) + # -- harmless (the inf it produces is always multiplied by 0 + # downstream) but not suppressed, unlike the equivalent case + # right below it in the same function. + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + bag = BagOfWords(features, 2_000, nstopwords=50, seed=0) # self.assertEqual(len(bag), 34_997) query = FileCollection("campus/holdout/*.png", mono=True)