Skip to content

Commit 24409c9

Browse files
authored
feat(ap): implement mobile graphql api (#35)
* feat(ap): implement mobile graphql api thumbnails are not working as-is, will fix this later * fix(ap): get thumbnails from blendedGallery element if available * chore(ap): resolve linter errors * fix(ap): exclude article footer * chore: remove unneeded rsshub item from config
1 parent 2d051d8 commit 24409c9

7 files changed

Lines changed: 345 additions & 225 deletions

File tree

‎config-example.xml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
<Config>
2-
<RSSHubAddress>http://127.0.0.1:1200</RSSHubAddress>
32
<SentryDSN></SentryDSN>
43
<IsDebug></IsDebug>
54
</Config>

‎generate_all_test.go‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
package main
22

33
import (
4-
"NewsChannel/news"
54
"bytes"
6-
"encoding/xml"
75
"fmt"
86
"hash/crc32"
97
"os"
@@ -96,20 +94,6 @@ func makeNews(_t *testing.T, hour int, dayDelta int) {
9694
}
9795

9896
func TestAllFileGeneration(_t *testing.T) {
99-
// Load config
100-
rawConfig, err := os.ReadFile("./config.xml")
101-
if err != nil {
102-
_t.Fatal(err)
103-
}
104-
105-
config := &Config{}
106-
err = xml.Unmarshal(rawConfig, config)
107-
if err != nil {
108-
_t.Fatal(err)
109-
}
110-
111-
news.RSSHubAddress = config.RSSHubAddress
112-
11397
t := time.Now()
11498

11599
for i := 0; i < t.Hour(); i++ {

‎main.go‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,9 @@ type News struct {
5757
}
5858

5959
type Config struct {
60-
XMLName xml.Name `xml:"Config"`
61-
RSSHubAddress string `xml:"RSSHubAddress"`
62-
SentryDSN string `xml:"SentryDSN"`
63-
IsDebug bool `xml:"IsDebug"`
60+
XMLName xml.Name `xml:"Config"`
61+
SentryDSN string `xml:"SentryDSN"`
62+
IsDebug bool `xml:"IsDebug"`
6463
}
6564

6665
var currentTime = 0
@@ -82,8 +81,6 @@ func main() {
8281
checkError(err)
8382
defer sentry.Flush(2 * time.Second)
8483

85-
news.RSSHubAddress = config.RSSHubAddress
86-
8784
// Load countries from JSON file
8885
countries, err := LoadCountries("countries.json")
8986
checkError(err)

‎news/ap/json.go‎

Lines changed: 275 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,275 @@
1+
package ap
2+
3+
import (
4+
"NewsChannel/news"
5+
"encoding/json"
6+
"errors"
7+
"fmt"
8+
"net/url"
9+
"regexp"
10+
"strings"
11+
12+
"github.com/PuerkitoBio/goquery"
13+
)
14+
15+
var BaseUrl = "https://apnews.com/graphql/delivery/ap/v1"
16+
17+
func (a *AP) getArticles(path string, topic news.Topic) ([]news.Article, error) {
18+
parsedURL, err := url.Parse(BaseUrl)
19+
if err != nil {
20+
return nil, err
21+
}
22+
query := parsedURL.Query()
23+
query.Add("operationName", "ContentPageQuery")
24+
query.Add("variables", fmt.Sprintf("{\"path\":\"%s\"}", path))
25+
query.Add("extensions", "{\"persistedQuery\":{\"version\":1,\"sha256Hash\":\"3bc305abbf62e9e632403a74cc86dc1cba51156d2313f09b3779efec51fc3acb\"}}")
26+
parsedURL.RawQuery = query.Encode()
27+
requestURL := parsedURL.String()
28+
29+
data, err := news.HttpGet(requestURL)
30+
if err != nil {
31+
return nil, err
32+
}
33+
34+
var categoryListing ContentPageQuery
35+
err = json.Unmarshal(data, &categoryListing)
36+
if err != nil {
37+
return nil, err
38+
}
39+
40+
var articles []news.Article
41+
for _, item := range categoryListing.Data.Screen.Main {
42+
if item.TypeName != "ColumnContainer" {
43+
continue
44+
}
45+
46+
for _, column := range item.Columns {
47+
if column.TypeName != "PageListModule" {
48+
continue
49+
}
50+
51+
for _, story := range column.Items {
52+
if story.TypeName != "PagePromo" {
53+
continue
54+
}
55+
56+
title := news.SanitizeText(story.Title)
57+
// Check for duplicates
58+
if news.IsDuplicateArticle(a.oldArticleTitles, title) {
59+
continue
60+
}
61+
a.oldArticleTitles = append(a.oldArticleTitles, title)
62+
63+
content, location, thumbnail, err := a.getFullArticle(story.Path)
64+
if err != nil {
65+
return nil, err
66+
}
67+
68+
if content == "" {
69+
continue
70+
}
71+
72+
article := news.Article{
73+
Title: title,
74+
Content: &content,
75+
Topic: topic,
76+
Location: location,
77+
Thumbnail: thumbnail,
78+
}
79+
80+
articles = append(articles, article)
81+
return articles, nil
82+
}
83+
}
84+
}
85+
86+
return articles, nil
87+
}
88+
89+
func (a *AP) getFullArticle(path string) (string, *news.Location, *news.Thumbnail, error) {
90+
if path == "" {
91+
return "", nil, nil, errors.New("empty path")
92+
}
93+
94+
parsedURL, err := url.Parse(BaseUrl)
95+
if err != nil {
96+
return "", nil, nil, err
97+
}
98+
query := parsedURL.Query()
99+
query.Add("operationName", "StoryQuery")
100+
query.Add("variables", fmt.Sprintf("{\"path\":\"%s\"}", path))
101+
query.Add("extensions", "{\"persistedQuery\":{\"version\":1,\"sha256Hash\":\"d61508dd5f8c1d84fa6d49338e321266f56b4538c07c233564bb11300287d69c\"}}")
102+
parsedURL.RawQuery = query.Encode()
103+
requestURL := parsedURL.String()
104+
105+
data, err := news.HttpGet(requestURL)
106+
if err != nil {
107+
return "", nil, nil, err
108+
}
109+
110+
var articleResponse StoryPageQuery
111+
err = json.Unmarshal(data, &articleResponse)
112+
if err != nil {
113+
return "", nil, nil, err
114+
}
115+
116+
// First, let's reconstruct the HTML that has been stored in the most horrific
117+
// format I've ever witnessed.
118+
var html strings.Builder
119+
for _, part := range articleResponse.Data.StoryPage.StoryBody {
120+
switch part.TypeName {
121+
case "HtmlString":
122+
html.WriteString(part.HTML)
123+
case "LinkEnhancement":
124+
for _, bodyPart := range part.Body {
125+
html.WriteString(bodyPart)
126+
}
127+
default:
128+
// We do not want other parts (video players, ads, etc)
129+
continue
130+
}
131+
}
132+
133+
// Convert the now usable HTML to a string for use in the channel, and
134+
// extract the location from the first paragraph.
135+
content, locationString, err := a.extractArticleBody(html.String())
136+
if err != nil {
137+
return "", nil, nil, err
138+
}
139+
if len(content) == 0 {
140+
return "", nil, nil, nil
141+
}
142+
143+
var location *news.Location
144+
if locationString != nil {
145+
location = news.GetLocationForExtractedLocation([]string{*locationString}, "en")
146+
}
147+
148+
thumbnail := a.downloadThumbnail(articleResponse)
149+
150+
return strings.TrimSpace(content), location, thumbnail, nil
151+
}
152+
153+
func (a *AP) extractArticleBody(html string) (string, *string, error) {
154+
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
155+
if err != nil {
156+
return "", nil, err
157+
}
158+
159+
var contentSlice []string
160+
161+
doc.Find("p").EachWithBreak(func(i int, elem *goquery.Selection) bool {
162+
text := strings.TrimSpace(elem.Text())
163+
if text == "___" {
164+
// We have reached the article footer
165+
return false
166+
}
167+
168+
if text != "" {
169+
contentSlice = append(contentSlice, text)
170+
}
171+
return true
172+
})
173+
174+
if len(contentSlice) == 0 {
175+
return "", nil, nil
176+
}
177+
178+
var content string
179+
for _, paragraph := range contentSlice {
180+
content += news.SanitizeText(paragraph)
181+
content += "\n\n"
182+
}
183+
184+
// Get the location
185+
locationRegex := regexp.MustCompile(`(.*?) \(AP\) — `)
186+
location := locationRegex.FindStringSubmatch(contentSlice[0])
187+
if len(location) > 1 && len(location[1]) > 0 {
188+
return content, &location[1], nil
189+
}
190+
191+
return content, nil, nil
192+
}
193+
194+
func (a *AP) downloadThumbnail(articleResponse StoryPageQuery) *news.Thumbnail {
195+
var imageURL string
196+
var caption string
197+
198+
if len(articleResponse.Data.StoryPage.BlendedGallery) > 0 {
199+
caption, imageURL = a.extractGalleryImage(articleResponse.Data.StoryPage.BlendedGallery)
200+
} else {
201+
caption, imageURL = a.extractLeadImage(articleResponse.Data.StoryPage.StoryLead)
202+
}
203+
204+
var imageData []byte
205+
if imageURL == "" {
206+
return nil
207+
}
208+
209+
imageData, err := news.HttpGet(imageURL)
210+
if err != nil || len(imageData) == 0 {
211+
return nil
212+
}
213+
214+
return &news.Thumbnail{
215+
Image: news.ConvertImage(imageData),
216+
Caption: news.SanitizeText(caption),
217+
}
218+
}
219+
220+
func (a *AP) extractGalleryImage(gallery []BlendedGalleryElement) (caption string, imageURL string) {
221+
for _, item := range gallery {
222+
if item.TypeName != "Carousel" {
223+
continue
224+
}
225+
226+
for _, slide := range item.Slides {
227+
if slide.TypeName != "GallerySlide" {
228+
continue
229+
}
230+
231+
for _, captionString := range slide.Caption {
232+
if len(captionString) > 0 {
233+
caption = captionString
234+
break
235+
}
236+
}
237+
238+
for _, media := range slide.Media {
239+
if media.TypeName != "Image" {
240+
continue
241+
}
242+
imageURL = a.extractURLFromMap(media.Image)
243+
return
244+
}
245+
}
246+
}
247+
return
248+
}
249+
250+
func (a *AP) extractLeadImage(storyLead []StoryLeadElement) (caption string, imageURL string) {
251+
for _, item := range storyLead {
252+
if item.TypeName != "Figure" {
253+
continue
254+
}
255+
256+
caption = item.AltText
257+
imageURL = a.extractURLFromMap(item.Image)
258+
return
259+
}
260+
261+
return
262+
}
263+
264+
func (a *AP) extractURLFromMap(imageMap ImageMap) string {
265+
if imageMap.TypeName != "Map" {
266+
return ""
267+
}
268+
for _, mapEntry := range imageMap.Entries {
269+
if mapEntry.TypeName != "MapEntry" || mapEntry.Key != "src" {
270+
continue
271+
}
272+
return mapEntry.Value
273+
}
274+
return ""
275+
}

0 commit comments

Comments
 (0)