Skip to content

Commit 1778358

Browse files
martin-gaievskimartin-gaievski
authored
Fix flaky HybridQueryExplainIT explanation test via per-_id assertions (#1900)
Signed-off-by: Martin Gaievski <gaievski@amazon.com>
1 parent a5e4e8f commit 1778358

2 files changed

Lines changed: 111 additions & 99 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
1515
* [Hybrid Query] Fix `hybrid_score_explanation` returning a single normalization block for hybrid queries on indices that contain a nested field ([#1876](https://github.com/opensearch-project/neural-search/pull/1876))
1616

1717
### Infrastructure
18+
* [Hybrid Query] Fix flaky `HybridQueryExplainIT` explanation test by asserting per-document explanation trees keyed by `_id` instead of hit position ([#1899](https://github.com/opensearch-project/neural-search/pull/1899))
1819

1920
### Documentation
2021

‎src/test/java/org/opensearch/neuralsearch/query/HybridQueryExplainIT.java‎

Lines changed: 110 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import java.util.ArrayList;
1717
import java.util.Arrays;
1818
import java.util.Collections;
19+
import java.util.HashMap;
1920
import java.util.List;
2021
import java.util.Locale;
2122
import java.util.Map;
@@ -319,107 +320,117 @@ public void testExplanationResponseProcessor_whenProcessorIsNotConfigured_thenRe
319320
assertEquals(RELATION_EQUAL_TO, total.get("relation"));
320321

321322
// explain
322-
Map<String, Object> searchHit1 = hitsNestedList.get(0);
323-
Map<String, Object> topLevelExplanationsHit1 = getValueByKey(searchHit1, "_explanation");
324-
assertNotNull(topLevelExplanationsHit1);
325-
assertEquals(0.343f, (double) topLevelExplanationsHit1.get("value"), DELTA_FOR_SCORE_ASSERTION);
323+
// Assert the per-document explanation trees keyed by _id rather than by hit position. Documents "1"
324+
// ("hello") and "2" ("place") both normalize to the same combined score (each is the sole/maximum match
325+
// of its own sub-query leg, so min_max maps both to 1.0 in-leg and they combine to an identical score),
326+
// so their relative rank is decided by a shard/segment tie-break and is not deterministic. Looking each
327+
// hit up by _id keeps every structural check while removing the ordering dependency.
328+
Map<String, Map<String, Object>> explanationById = new HashMap<>();
329+
for (Map<String, Object> oneHit : hitsNestedList) {
330+
explanationById.put((String) oneHit.get("_id"), getValueByKey(oneHit, "_explanation"));
331+
}
326332
String expectedTopLevelDescription = "combined score of:";
327-
assertEquals(expectedTopLevelDescription, topLevelExplanationsHit1.get("description"));
328-
List<Map<String, Object>> normalizationExplanationHit1 = getListOfValues(topLevelExplanationsHit1, "details");
329-
assertEquals(2, normalizationExplanationHit1.size());
330-
331-
Map<String, Object> noMatchDetailsForHit1 = normalizationExplanationHit1.get(0);
332-
assertEquals(0.0f, (double) noMatchDetailsForHit1.get("value"), DELTA_FOR_SCORE_ASSERTION);
333-
assertEquals("no matching term", noMatchDetailsForHit1.get("description"));
334-
assertEquals(0, ((List) noMatchDetailsForHit1.get("details")).size());
335-
336-
Map<String, Object> hit1DetailsForHit1 = normalizationExplanationHit1.get(1);
337-
assertEquals(0.343f, (double) hit1DetailsForHit1.get("value"), DELTA_FOR_SCORE_ASSERTION);
338-
assertEquals("sum of:", hit1DetailsForHit1.get("description"));
339-
assertEquals(1, ((List) hit1DetailsForHit1.get("details")).size());
340-
341-
Map<String, Object> explanationsHit1 = getListOfValues(hit1DetailsForHit1, "details").get(0);
342-
assertEquals("weight(test-text-field-1:place in 0) [PerFieldSimilarity], result of:", explanationsHit1.get("description"));
343-
assertEquals(0.343f, (double) explanationsHit1.get("value"), DELTA_FOR_SCORE_ASSERTION);
344-
assertEquals(1, ((List) explanationsHit1.get("details")).size());
345-
346-
Map<String, Object> explanationsHit1Details = getListOfValues(explanationsHit1, "details").get(0);
347-
assertEquals(0.343f, (double) explanationsHit1Details.get("value"), DELTA_FOR_SCORE_ASSERTION);
348-
assertEquals("score(freq=1.0), computed as boost * idf * tf from:", explanationsHit1Details.get("description"));
349-
assertEquals(2, getListOfValues(explanationsHit1Details, "details").size());
350-
351-
Map<String, Object> explanationsDetails2Hit1Details = getListOfValues(explanationsHit1Details, "details").get(0);
352-
assertEquals(0.693f, (double) explanationsDetails2Hit1Details.get("value"), DELTA_FOR_SCORE_ASSERTION);
353-
assertEquals("idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:", explanationsDetails2Hit1Details.get("description"));
354-
assertFalse(getListOfValues(explanationsDetails2Hit1Details, "details").isEmpty());
355-
356-
Map<String, Object> explanationsDetails3Hit1Details = getListOfValues(explanationsHit1Details, "details").get(1);
357-
assertEquals(0.495f, (double) explanationsDetails3Hit1Details.get("value"), DELTA_FOR_SCORE_ASSERTION);
358-
assertEquals(
359-
"tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
360-
explanationsDetails3Hit1Details.get("description")
361-
);
362-
assertFalse(getListOfValues(explanationsDetails3Hit1Details, "details").isEmpty());
363-
364-
// search hit 2
365-
Map<String, Object> searchHit2 = hitsNestedList.get(1);
366-
Map<String, Object> topLevelExplanationsHit2 = getValueByKey(searchHit2, "_explanation");
367-
assertNotNull(topLevelExplanationsHit2);
368-
assertEquals(0.13f, (double) topLevelExplanationsHit2.get("value"), DELTA_FOR_SCORE_ASSERTION);
369-
370-
assertEquals(expectedTopLevelDescription, topLevelExplanationsHit2.get("description"));
371-
List<Map<String, Object>> normalizationExplanationHit2 = getListOfValues(topLevelExplanationsHit2, "details");
372-
assertEquals(2, normalizationExplanationHit2.size());
373333

374-
Map<String, Object> hit1DetailsForHit2 = normalizationExplanationHit2.get(0);
375-
assertEquals(0.13f, (double) hit1DetailsForHit2.get("value"), DELTA_FOR_SCORE_ASSERTION);
376-
assertEquals("weight(test-text-field-1:hello in 0) [PerFieldSimilarity], result of:", hit1DetailsForHit2.get("description"));
377-
assertEquals(1, getListOfValues(hit1DetailsForHit2, "details").size());
378-
379-
Map<String, Object> explanationsHit2 = getListOfValues(hit1DetailsForHit2, "details").get(0);
380-
assertEquals(0.13f, (double) explanationsHit2.get("value"), DELTA_FOR_SCORE_ASSERTION);
381-
assertEquals("score(freq=1.0), computed as boost * idf * tf from:", explanationsHit2.get("description"));
382-
assertEquals(2, getListOfValues(explanationsHit2, "details").size());
383-
384-
Map<String, Object> explanationsHit2Details = getListOfValues(explanationsHit2, "details").get(1);
385-
assertEquals(0.454f, (double) explanationsHit2Details.get("value"), DELTA_FOR_SCORE_ASSERTION);
386-
assertEquals("tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:", explanationsHit2Details.get("description"));
387-
assertEquals(5, getListOfValues(explanationsHit2Details, "details").size());
388-
389-
Map<String, Object> hit1DetailsForHit2NoMatch = normalizationExplanationHit2.get(0);
390-
assertEquals(0.13f, (double) hit1DetailsForHit2NoMatch.get("value"), DELTA_FOR_SCORE_ASSERTION);
391-
assertEquals("weight(test-text-field-1:hello in 0) [PerFieldSimilarity], result of:", hit1DetailsForHit2NoMatch.get("description"));
392-
assertEquals(1, ((List) hit1DetailsForHit2NoMatch.get("details")).size());
393-
394-
// search hit 3
395-
Map<String, Object> searchHit3 = hitsNestedList.get(1);
396-
Map<String, Object> topLevelExplanationsHit3 = getValueByKey(searchHit3, "_explanation");
397-
assertNotNull(topLevelExplanationsHit3);
398-
assertEquals(0.13f, (double) topLevelExplanationsHit3.get("value"), DELTA_FOR_SCORE_ASSERTION);
399-
400-
assertEquals(expectedTopLevelDescription, topLevelExplanationsHit3.get("description"));
401-
List<Map<String, Object>> normalizationExplanationHit3 = getListOfValues(topLevelExplanationsHit3, "details");
402-
assertEquals(2, normalizationExplanationHit3.size());
403-
404-
Map<String, Object> hit1DetailsForHit3 = normalizationExplanationHit3.get(0);
405-
assertEquals(0.13f, (double) hit1DetailsForHit3.get("value"), DELTA_FOR_SCORE_ASSERTION);
406-
assertEquals("weight(test-text-field-1:hello in 0) [PerFieldSimilarity], result of:", hit1DetailsForHit3.get("description"));
407-
assertEquals(1, getListOfValues(hit1DetailsForHit3, "details").size());
408-
409-
Map<String, Object> explanationsHit3 = getListOfValues(hit1DetailsForHit3, "details").get(0);
410-
assertEquals(0.13f, (double) explanationsHit3.get("value"), DELTA_FOR_SCORE_ASSERTION);
411-
assertEquals("score(freq=1.0), computed as boost * idf * tf from:", explanationsHit3.get("description"));
412-
assertEquals(2, getListOfValues(explanationsHit3, "details").size());
413-
414-
Map<String, Object> explanationsHit3Details = getListOfValues(explanationsHit3, "details").get(0);
415-
assertEquals(0.287f, (double) explanationsHit3Details.get("value"), DELTA_FOR_SCORE_ASSERTION);
416-
assertEquals("idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:", explanationsHit3Details.get("description"));
417-
assertEquals(2, getListOfValues(explanationsHit3Details, "details").size());
418-
419-
Map<String, Object> hit1DetailsForHit3NoMatch = normalizationExplanationHit2.get(1);
420-
assertEquals(0.0f, (double) hit1DetailsForHit3NoMatch.get("value"), DELTA_FOR_SCORE_ASSERTION);
421-
assertEquals("No matching clauses", hit1DetailsForHit3NoMatch.get("description"));
422-
assertEquals(0, ((List) hit1DetailsForHit3NoMatch.get("details")).size());
334+
// doc "2" ("Hi to this place"): sub-query 0 ("hello") does not match; sub-query 1 matches on "place"
335+
Map<String, Object> explanationDoc2 = explanationById.get("2");
336+
assertNotNull(explanationDoc2);
337+
assertEquals(0.343f, (double) explanationDoc2.get("value"), DELTA_FOR_SCORE_ASSERTION);
338+
assertEquals(expectedTopLevelDescription, explanationDoc2.get("description"));
339+
List<Map<String, Object>> subQueriesDoc2 = getListOfValues(explanationDoc2, "details");
340+
assertEquals(2, subQueriesDoc2.size());
341+
342+
Map<String, Object> doc2SubQuery0 = subQueriesDoc2.get(0);
343+
assertEquals(0.0f, (double) doc2SubQuery0.get("value"), DELTA_FOR_SCORE_ASSERTION);
344+
assertEquals("no matching term", doc2SubQuery0.get("description"));
345+
assertEquals(0, getListOfValues(doc2SubQuery0, "details").size());
346+
347+
Map<String, Object> doc2SubQuery1 = subQueriesDoc2.get(1);
348+
assertEquals(0.343f, (double) doc2SubQuery1.get("value"), DELTA_FOR_SCORE_ASSERTION);
349+
assertEquals("sum of:", doc2SubQuery1.get("description"));
350+
assertEquals(1, getListOfValues(doc2SubQuery1, "details").size());
351+
352+
Map<String, Object> doc2Weight = getListOfValues(doc2SubQuery1, "details").get(0);
353+
assertEquals("weight(test-text-field-1:place in 0) [PerFieldSimilarity], result of:", doc2Weight.get("description"));
354+
assertEquals(0.343f, (double) doc2Weight.get("value"), DELTA_FOR_SCORE_ASSERTION);
355+
assertEquals(1, getListOfValues(doc2Weight, "details").size());
356+
357+
Map<String, Object> doc2Score = getListOfValues(doc2Weight, "details").get(0);
358+
assertEquals(0.343f, (double) doc2Score.get("value"), DELTA_FOR_SCORE_ASSERTION);
359+
assertEquals("score(freq=1.0), computed as boost * idf * tf from:", doc2Score.get("description"));
360+
assertEquals(2, getListOfValues(doc2Score, "details").size());
361+
362+
Map<String, Object> doc2Idf = getListOfValues(doc2Score, "details").get(0);
363+
assertEquals(0.693f, (double) doc2Idf.get("value"), DELTA_FOR_SCORE_ASSERTION);
364+
assertEquals("idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:", doc2Idf.get("description"));
365+
assertFalse(getListOfValues(doc2Idf, "details").isEmpty());
366+
367+
Map<String, Object> doc2Tf = getListOfValues(doc2Score, "details").get(1);
368+
assertEquals(0.495f, (double) doc2Tf.get("value"), DELTA_FOR_SCORE_ASSERTION);
369+
assertEquals("tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:", doc2Tf.get("description"));
370+
assertFalse(getListOfValues(doc2Tf, "details").isEmpty());
371+
372+
// doc "1" ("Hello world"): sub-query 0 matches on "hello"; sub-query 1 ("place"/"welcome") does not match
373+
Map<String, Object> explanationDoc1 = explanationById.get("1");
374+
assertNotNull(explanationDoc1);
375+
assertEquals(0.13f, (double) explanationDoc1.get("value"), DELTA_FOR_SCORE_ASSERTION);
376+
assertEquals(expectedTopLevelDescription, explanationDoc1.get("description"));
377+
List<Map<String, Object>> subQueriesDoc1 = getListOfValues(explanationDoc1, "details");
378+
assertEquals(2, subQueriesDoc1.size());
379+
380+
Map<String, Object> doc1SubQuery0 = subQueriesDoc1.get(0);
381+
assertEquals(0.13f, (double) doc1SubQuery0.get("value"), DELTA_FOR_SCORE_ASSERTION);
382+
assertEquals("weight(test-text-field-1:hello in 0) [PerFieldSimilarity], result of:", doc1SubQuery0.get("description"));
383+
assertEquals(1, getListOfValues(doc1SubQuery0, "details").size());
384+
385+
Map<String, Object> doc1Score = getListOfValues(doc1SubQuery0, "details").get(0);
386+
assertEquals(0.13f, (double) doc1Score.get("value"), DELTA_FOR_SCORE_ASSERTION);
387+
assertEquals("score(freq=1.0), computed as boost * idf * tf from:", doc1Score.get("description"));
388+
assertEquals(2, getListOfValues(doc1Score, "details").size());
389+
390+
// sub-query 1 did not match this document; assert value and description only. The number of child
391+
// "no match on optional clause" entries under "No matching clauses" is a Lucene BooleanQuery rendering
392+
// detail (one per optional clause) and is not part of the hybrid explanation contract.
393+
Map<String, Object> doc1SubQuery1 = subQueriesDoc1.get(1);
394+
assertEquals(0.0f, (double) doc1SubQuery1.get("value"), DELTA_FOR_SCORE_ASSERTION);
395+
assertEquals("No matching clauses", doc1SubQuery1.get("description"));
396+
397+
// doc "3" ("We would like to welcome everyone"): sub-query 0 does not match; sub-query 1 matches on "welcome"
398+
Map<String, Object> explanationDoc3 = explanationById.get("3");
399+
assertNotNull(explanationDoc3);
400+
assertEquals(0.291f, (double) explanationDoc3.get("value"), DELTA_FOR_SCORE_ASSERTION);
401+
assertEquals(expectedTopLevelDescription, explanationDoc3.get("description"));
402+
List<Map<String, Object>> subQueriesDoc3 = getListOfValues(explanationDoc3, "details");
403+
assertEquals(2, subQueriesDoc3.size());
404+
405+
Map<String, Object> doc3SubQuery0 = subQueriesDoc3.get(0);
406+
assertEquals(0.0f, (double) doc3SubQuery0.get("value"), DELTA_FOR_SCORE_ASSERTION);
407+
assertEquals("no matching term", doc3SubQuery0.get("description"));
408+
assertEquals(0, getListOfValues(doc3SubQuery0, "details").size());
409+
410+
Map<String, Object> doc3SubQuery1 = subQueriesDoc3.get(1);
411+
assertEquals(0.291f, (double) doc3SubQuery1.get("value"), DELTA_FOR_SCORE_ASSERTION);
412+
assertEquals("sum of:", doc3SubQuery1.get("description"));
413+
assertEquals(1, getListOfValues(doc3SubQuery1, "details").size());
414+
415+
Map<String, Object> doc3Weight = getListOfValues(doc3SubQuery1, "details").get(0);
416+
assertEquals("weight(test-text-field-1:welcome in 0) [PerFieldSimilarity], result of:", doc3Weight.get("description"));
417+
assertEquals(0.291f, (double) doc3Weight.get("value"), DELTA_FOR_SCORE_ASSERTION);
418+
assertEquals(1, getListOfValues(doc3Weight, "details").size());
419+
420+
Map<String, Object> doc3Score = getListOfValues(doc3Weight, "details").get(0);
421+
assertEquals(0.291f, (double) doc3Score.get("value"), DELTA_FOR_SCORE_ASSERTION);
422+
assertEquals("score(freq=1.0), computed as boost * idf * tf from:", doc3Score.get("description"));
423+
assertEquals(2, getListOfValues(doc3Score, "details").size());
424+
425+
Map<String, Object> doc3Idf = getListOfValues(doc3Score, "details").get(0);
426+
assertEquals(0.693f, (double) doc3Idf.get("value"), DELTA_FOR_SCORE_ASSERTION);
427+
assertEquals("idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:", doc3Idf.get("description"));
428+
assertFalse(getListOfValues(doc3Idf, "details").isEmpty());
429+
430+
Map<String, Object> doc3Tf = getListOfValues(doc3Score, "details").get(1);
431+
assertEquals(0.420f, (double) doc3Tf.get("value"), DELTA_FOR_SCORE_ASSERTION);
432+
assertEquals("tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:", doc3Tf.get("description"));
433+
assertFalse(getListOfValues(doc3Tf, "details").isEmpty());
423434
}
424435

425436
@SneakyThrows

0 commit comments

Comments
 (0)