Skip to content

Commit 2b744e3

Browse files
committed
test: cover base64 placeholder, figure synthesis, and media counts
Round out media.go coverage with focused unit tests: fixLazyImages (e2e via FromReader): - tiny base64 placeholder + sibling data-src -> placeholder dropped, real URL promoted to src - SVG base64 data URL preserved (may legitimately be the real image) - large base64 (>= 133 char body) preserved even with sibling data-src - <figure> with no <img> child synthesizes one from a data-* image URL preservableMediaCount / removableEmbedCount (direct calls): - audio with src counts; bare audio does not - video with blob: src counts - video with data-video-id counts - iframe matching the video allow-list counts as preservable, NOT as removable - unrecognized iframe goes the other way - removableEmbedCount picks up object / embed / iframe alike - inner-HTML allow path: <object><param value=...>youtube</param></object> Coverage delta (default suite): fixLazyImages 51.3% -> 97.4% preservableMediaCount 17.6% -> 94.1% removableEmbedCount 23.1% -> 100.0% total 76.8% -> 78.6%
1 parent 7fc477a commit 2b744e3

2 files changed

Lines changed: 255 additions & 0 deletions

File tree

‎media_clean_test.go‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,3 +129,87 @@ func TestReplaceJavascriptLinksLeavesNormalAnchors(t *testing.T) {
129129
t.Fatalf("normal anchor href should be preserved, got: %s", article.Content)
130130
}
131131
}
132+
133+
// TestFixLazyImagesDropsTinyBase64Placeholder exercises the first pass of
134+
// fixLazyImages: when src is a small (< 133 chars beyond prefix) base64
135+
// data URL AND another attribute on the same element looks like an image
136+
// URL, the placeholder src is dropped so pass 2 can promote the data-*
137+
// URL into src.
138+
func TestFixLazyImagesDropsTinyBase64Placeholder(t *testing.T) {
139+
html := `<html><body><article>
140+
<p>This article body has enough useful text to clear the extractor length checks while we exercise the base64 placeholder removal branch and verify the real image URL stored in a sibling data attribute is promoted to src.</p>
141+
<p>A second paragraph keeps the candidate substantial during conditional cleanup so the recovered image survives long enough for assertions on the final HTML output.</p>
142+
<img src="data:image/gif;base64,R0lGODlhAQABAAAAACw=" data-src="https://cdn.example.com/real-photo.jpg">
143+
</article></body></html>`
144+
145+
article, err := FromReader(strings.NewReader(html), "", nil)
146+
if err != nil {
147+
t.Fatalf("FromReader: %v", err)
148+
}
149+
if strings.Contains(article.Content, "data:image/gif;base64") {
150+
t.Fatalf("tiny base64 placeholder should be removed, got: %s", article.Content)
151+
}
152+
if !strings.Contains(article.Content, `src="https://cdn.example.com/real-photo.jpg"`) {
153+
t.Fatalf("expected data-src to be promoted to src, got: %s", article.Content)
154+
}
155+
}
156+
157+
// TestFixLazyImagesPreservesSVGDataURL ensures SVG base64 data URLs are
158+
// kept as-is — they may legitimately be the real image rather than a
159+
// placeholder.
160+
func TestFixLazyImagesPreservesSVGDataURL(t *testing.T) {
161+
html := `<html><body><article>
162+
<p>This article body has enough useful text to clear the extractor length checks while we verify SVG data URLs are preserved by the base64 placeholder branch even when other image-shaped attributes are present on the same element.</p>
163+
<p>A second paragraph keeps the candidate substantial during conditional cleanup so the SVG image survives long enough for assertions to inspect the final HTML output.</p>
164+
<img src="data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciLz4=" data-src="https://cdn.example.com/other.jpg">
165+
</article></body></html>`
166+
167+
article, err := FromReader(strings.NewReader(html), "", nil)
168+
if err != nil {
169+
t.Fatalf("FromReader: %v", err)
170+
}
171+
if !strings.Contains(article.Content, "data:image/svg+xml") {
172+
t.Fatalf("SVG data URL must be preserved, got: %s", article.Content)
173+
}
174+
}
175+
176+
// TestFixLazyImagesPreservesLargeBase64 confirms large base64 data URLs
177+
// (≥ 133 chars beyond the prefix) are NOT removed even when a sibling
178+
// data-* attribute carries an image URL — the heuristic only targets the
179+
// tiny placeholders typical of lazy-loading templates.
180+
func TestFixLazyImagesPreservesLargeBase64(t *testing.T) {
181+
// 200 chars of base64 payload, well above the 133-char threshold.
182+
largePayload := strings.Repeat("A", 200)
183+
html := `<html><body><article>
184+
<p>This article body has enough useful text to clear the extractor length checks while we verify a large base64 data URL is left alone by the placeholder removal heuristic, since it is presumably the real image.</p>
185+
<p>A second paragraph keeps the candidate substantial during conditional cleanup so the image survives long enough for assertions to inspect the final HTML output verifying preservation.</p>
186+
<img src="data:image/png;base64,` + largePayload + `" data-src="https://cdn.example.com/other.jpg">
187+
</article></body></html>`
188+
189+
article, err := FromReader(strings.NewReader(html), "", nil)
190+
if err != nil {
191+
t.Fatalf("FromReader: %v", err)
192+
}
193+
if !strings.Contains(article.Content, "data:image/png;base64") {
194+
t.Fatalf("large base64 data URL must be preserved, got: %s", article.Content)
195+
}
196+
}
197+
198+
// TestFixLazyImagesFigureSynthesizesImg verifies the <figure> branch:
199+
// when a figure has no <img>/<picture> child but carries a data-* image
200+
// URL, the recovery pass synthesizes a child <img> with src populated.
201+
func TestFixLazyImagesFigureSynthesizesImg(t *testing.T) {
202+
html := `<html><body><article>
203+
<p>This article body has enough useful text to clear the extractor length checks while we exercise the figure recovery branch that synthesizes a child img element when a data attribute carries a single image URL.</p>
204+
<p>A second paragraph keeps the candidate substantial during conditional cleanup so the synthesized image survives long enough for assertions to inspect the final HTML output.</p>
205+
<figure data-src="https://cdn.example.com/synthesized.jpg"><figcaption>caption text</figcaption></figure>
206+
</article></body></html>`
207+
208+
article, err := FromReader(strings.NewReader(html), "", nil)
209+
if err != nil {
210+
t.Fatalf("FromReader: %v", err)
211+
}
212+
if !strings.Contains(article.Content, `src="https://cdn.example.com/synthesized.jpg"`) {
213+
t.Fatalf("expected synthesized <img> with src populated, got: %s", article.Content)
214+
}
215+
}

‎media_count_test.go‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
package readability
2+
3+
import (
4+
"strings"
5+
"testing"
6+
7+
"github.com/PuerkitoBio/goquery"
8+
)
9+
10+
// TestPreservableMediaCountAudioWithSrc covers the <audio src=…> branch:
11+
// an audio element with a source URL counts as preservable media even
12+
// when no other attribute matches the video allow-list.
13+
func TestPreservableMediaCountAudioWithSrc(t *testing.T) {
14+
doc, err := goquery.NewDocumentFromReader(strings.NewReader(
15+
`<div><audio src="https://example.com/podcast.mp3"></audio></div>`))
16+
if err != nil {
17+
t.Fatal(err)
18+
}
19+
cfg := defaultParserConfig()
20+
got := preservableMediaCount(doc.Find("div"), cfg)
21+
if got != 1 {
22+
t.Fatalf("preservableMediaCount = %d, want 1 (audio with src)", got)
23+
}
24+
}
25+
26+
// TestPreservableMediaCountAudioWithoutSrc verifies the audio branch
27+
// requires a non-empty src; bare <audio> tags do not count.
28+
func TestPreservableMediaCountAudioWithoutSrc(t *testing.T) {
29+
doc, err := goquery.NewDocumentFromReader(strings.NewReader(`<div><audio></audio></div>`))
30+
if err != nil {
31+
t.Fatal(err)
32+
}
33+
cfg := defaultParserConfig()
34+
got := preservableMediaCount(doc.Find("div"), cfg)
35+
if got != 0 {
36+
t.Fatalf("preservableMediaCount = %d, want 0 (audio without src)", got)
37+
}
38+
}
39+
40+
// TestPreservableMediaCountVideoBlobSrc covers the video blob: branch:
41+
// player elements often surface their bound stream as a blob: URL, which
42+
// must be treated as legitimate media even though the URL itself is
43+
// useless to readers.
44+
func TestPreservableMediaCountVideoBlobSrc(t *testing.T) {
45+
doc, err := goquery.NewDocumentFromReader(strings.NewReader(
46+
`<div><video src="blob:https://example.com/abc-123"></video></div>`))
47+
if err != nil {
48+
t.Fatal(err)
49+
}
50+
cfg := defaultParserConfig()
51+
got := preservableMediaCount(doc.Find("div"), cfg)
52+
if got != 1 {
53+
t.Fatalf("preservableMediaCount = %d, want 1 (video blob:)", got)
54+
}
55+
}
56+
57+
// TestPreservableMediaCountVideoDataVideoID covers the data-video-id
58+
// branch: many CMS templates render an empty <video> shell that gets
59+
// hydrated by JS using data-video-id; this must still count as media.
60+
func TestPreservableMediaCountVideoDataVideoID(t *testing.T) {
61+
doc, err := goquery.NewDocumentFromReader(strings.NewReader(
62+
`<div><video data-video-id="abc123"></video></div>`))
63+
if err != nil {
64+
t.Fatal(err)
65+
}
66+
cfg := defaultParserConfig()
67+
got := preservableMediaCount(doc.Find("div"), cfg)
68+
if got != 1 {
69+
t.Fatalf("preservableMediaCount = %d, want 1 (video data-video-id)", got)
70+
}
71+
}
72+
73+
// TestPreservableMediaCountIframeAllowed covers the generic allow-list
74+
// branch: an iframe whose src matches cfg.videoAllowed counts as
75+
// preservable media.
76+
func TestPreservableMediaCountIframeAllowed(t *testing.T) {
77+
doc, err := goquery.NewDocumentFromReader(strings.NewReader(
78+
`<div><iframe src="https://www.youtube.com/embed/abc"></iframe></div>`))
79+
if err != nil {
80+
t.Fatal(err)
81+
}
82+
cfg := defaultParserConfig()
83+
got := preservableMediaCount(doc.Find("div"), cfg)
84+
if got != 1 {
85+
t.Fatalf("preservableMediaCount = %d, want 1 (iframe in allow-list)", got)
86+
}
87+
}
88+
89+
// TestPreservableMediaCountIframeUnrecognized verifies that iframes
90+
// whose attributes do NOT match the allow-list are not counted as
91+
// preservable media — they fall through to removableEmbedCount instead.
92+
func TestPreservableMediaCountIframeUnrecognized(t *testing.T) {
93+
doc, err := goquery.NewDocumentFromReader(strings.NewReader(
94+
`<div><iframe src="https://ads.example.com/banner"></iframe></div>`))
95+
if err != nil {
96+
t.Fatal(err)
97+
}
98+
cfg := defaultParserConfig()
99+
got := preservableMediaCount(doc.Find("div"), cfg)
100+
if got != 0 {
101+
t.Fatalf("preservableMediaCount = %d, want 0 (unrecognized iframe)", got)
102+
}
103+
}
104+
105+
// TestRemovableEmbedCountUnrecognized confirms that an iframe outside
106+
// the allow-list bumps removable embed count, which conditional cleanup
107+
// uses to push the candidate toward removal.
108+
func TestRemovableEmbedCountUnrecognized(t *testing.T) {
109+
doc, err := goquery.NewDocumentFromReader(strings.NewReader(
110+
`<div><iframe src="https://ads.example.com/banner"></iframe></div>`))
111+
if err != nil {
112+
t.Fatal(err)
113+
}
114+
cfg := defaultParserConfig()
115+
got := removableEmbedCount(doc.Find("div"), cfg)
116+
if got != 1 {
117+
t.Fatalf("removableEmbedCount = %d, want 1 (unrecognized iframe)", got)
118+
}
119+
}
120+
121+
// TestRemovableEmbedCountAllowedAttribute confirms that an iframe whose
122+
// attribute matches the allow-list is NOT counted — preservableMediaCount
123+
// will pick it up instead.
124+
func TestRemovableEmbedCountAllowedAttribute(t *testing.T) {
125+
doc, err := goquery.NewDocumentFromReader(strings.NewReader(
126+
`<div><iframe src="https://www.youtube.com/embed/abc"></iframe></div>`))
127+
if err != nil {
128+
t.Fatal(err)
129+
}
130+
cfg := defaultParserConfig()
131+
got := removableEmbedCount(doc.Find("div"), cfg)
132+
if got != 0 {
133+
t.Fatalf("removableEmbedCount = %d, want 0 (iframe in allow-list)", got)
134+
}
135+
}
136+
137+
// TestRemovableEmbedCountAllowedInnerHTML covers the inner-HTML allow
138+
// path: an embed/object that holds the recognized URL inside a child
139+
// node rather than on its own attributes (e.g. <object><param value=…/>
140+
// </object>) must also be considered allowed.
141+
func TestRemovableEmbedCountAllowedInnerHTML(t *testing.T) {
142+
doc, err := goquery.NewDocumentFromReader(strings.NewReader(
143+
`<div><object><param name="movie" value="https://www.youtube.com/embed/abc"/></object></div>`))
144+
if err != nil {
145+
t.Fatal(err)
146+
}
147+
cfg := defaultParserConfig()
148+
got := removableEmbedCount(doc.Find("div"), cfg)
149+
if got != 0 {
150+
t.Fatalf("removableEmbedCount = %d, want 0 (allowed via inner HTML)", got)
151+
}
152+
}
153+
154+
// TestRemovableEmbedCountObjectAndEmbed covers the multi-element path:
155+
// object and embed tags are inspected alongside iframe.
156+
func TestRemovableEmbedCountObjectAndEmbed(t *testing.T) {
157+
doc, err := goquery.NewDocumentFromReader(strings.NewReader(
158+
`<div>
159+
<object data="https://ads.example.com/x"></object>
160+
<embed src="https://ads.example.com/y">
161+
<iframe src="https://ads.example.com/z"></iframe>
162+
</div>`))
163+
if err != nil {
164+
t.Fatal(err)
165+
}
166+
cfg := defaultParserConfig()
167+
got := removableEmbedCount(doc.Find("div"), cfg)
168+
if got != 3 {
169+
t.Fatalf("removableEmbedCount = %d, want 3 (three unrecognized embeds)", got)
170+
}
171+
}

0 commit comments

Comments
 (0)