fix: headers for reuters requests (#37)

this does not fix reuters web api for japan, just for mobile. refactored the HttpGet function to take headers rather than a useragent string

Author: hwalker56

news/ansa/json.go

@@ -31,9 +31,13 @@ type Item struct {
 	GUID        string `xml:"guid"`
 }
 
+var Headers = map[string]string{
+	"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+}
+
 func (a *ANSA) getArticles(url string, topic news.Topic) ([]news.Article, error) {
 	// Fetch RSS XML
-	data, err := news.HttpGet(url, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+	data, err := news.HttpGet(url, Headers)
 	if err != nil {
 		return nil, err
 	}
@@ -93,7 +97,7 @@ func (a *ANSA) getFullArticle(articleURL string) (content string, location *news
 		return "", nil, nil
 	}
 
-	data, err := news.HttpGet(articleURL, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+	data, err := news.HttpGet(articleURL, Headers)
 	if err != nil {
 		log.Printf("Failed to fetch article content from %s: %v", articleURL, err)
 		return "", nil, nil
@@ -200,7 +204,7 @@ func (a *ANSA) extractThumbnail(html string) *news.Thumbnail {
 		return nil
 	}
 
-	imageData, err := news.HttpGet(imageURL, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+	imageData, err := news.HttpGet(imageURL, Headers)
 	if err != nil || len(imageData) == 0 {
 		return nil
 	}

news/reuters/json.go

@@ -10,8 +10,15 @@ import (
 	"strings"
 )
 
+var Headers = map[string]string{
+	"content-type":         "application/json",
+	"user-agent":           "ReutersNews/7.48.0.1790290736 Mozilla/5.0 (Linux; Android 14; SM-A135F Build/UP1A.231005.007; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/153.0.8010.36 Mobile Safari/537.36",
+	"accept":               "application/json",
+	"x-datadog-tracked-by": "react-native",
+}
+
 func (r *Reuters) getMobileArticles(url string, topic news.Topic) ([]news.Article, error) {
-	data, err := news.HttpGet(fmt.Sprintf("https://www.reuters.com/mobile/v1%s?outputType=json",url), "ReutersNews/7.6.0 iPad8,6 iPadOS/18.1 CFNetwork/1.0 Darwin/24.1.0")
+	data, err := news.HttpGet(fmt.Sprintf("https://www.reuters.com/mobile/v1%s?outputType=json", url), Headers)
 	if err != nil {
 		return nil, err
 	}
@@ -53,7 +60,7 @@ func (r *Reuters) getMobileArticles(url string, topic news.Topic) ([]news.Articl
 			// The mobile API is much easier to parse.
 			articlePath := story.URL
 			articleURL := fmt.Sprintf("https://www.reuters.com/mobile/v1%s", articlePath)
-			articleData, err := news.HttpGet(articleURL, "ReutersNews/7.6.0 iPad8,6 iPadOS/18.1 CFNetwork/1.0 Darwin/24.1.0")
+			articleData, err := news.HttpGet(articleURL, Headers)
 			if err != nil {
 				return nil, err
 			}
@@ -93,7 +100,7 @@ func (r *Reuters) getMobileArticles(url string, topic news.Topic) ([]news.Articl
 }
 
 func (r *Reuters) getWebArticles(url string, topic news.Topic) ([]news.Article, error) {
-	data, err := news.HttpGet(fmt.Sprintf("https://jp.reuters.com/pf/api/v3/content/fetch/articles-by-section-alias-or-id-v1?query={\"fetch_type\":\"collection_or_section\",\"orderby\":\"last_updated_date:desc\",\"section_id\":\"%s\",\"website\":\"reuters-japan\"}", url))
+	data, err := news.HttpGet(fmt.Sprintf("https://www.reuters.com/pf/api/v3/content/fetch/articles-by-section-alias-or-id-v1?query={\"fetch_type\":\"collection_or_section\",\"orderby\":\"last_updated_date:desc\",\"section_id\":\"%s\",\"website\":\"reuters-japan\"}", url))
 	if err != nil {
 		return nil, err
 	}
@@ -116,7 +123,7 @@ func (r *Reuters) getWebArticles(url string, topic news.Topic) ([]news.Article,
 		r.oldArticleTitles = append(r.oldArticleTitles, title)
 
 		articlePath := story.CanonicalURL
-		articleURL := fmt.Sprintf("https://jp.reuters.com/pf/api/v3/content/fetch/article-by-id-or-url-v1?query={\"website_url\":\"%s\",\"website\":\"reuters-japan\"}", articlePath)
+		articleURL := fmt.Sprintf("https://www.reuters.com/pf/api/v3/content/fetch/article-by-id-or-url-v1?query={\"website_url\":\"%s\",\"website\":\"reuters-japan\"}", articlePath)
 		articleData, err := news.HttpGet(articleURL)
 		if err != nil {
 			return nil, err
@@ -206,7 +213,7 @@ func getThumbnail(thumbnail ReutersThumbnail) (*news.Thumbnail, error) {
 		return nil, nil
 	}
 
-	data, err := news.HttpGet(thumbnail.URL, "ReutersNews/7.6.0 iPad8,6 iPadOS/18.1 CFNetwork/1.0 Darwin/24.1.0")
+	data, err := news.HttpGet(thumbnail.URL, Headers)
 	if err != nil {
 		return nil, err
 	}

news/utils.go

@@ -23,16 +23,18 @@ import (
 	_ "image/png"
 )
 
-func HttpGet(url string, userAgent ...string) ([]byte, error) {
+func HttpGet(url string, headers ...map[string]string) ([]byte, error) {
 	client := &http.Client{}
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return nil, err
 	}
 	req.Header.Set("User-Agent", "WiiLink News Channel File Generator")
 
-	if len(userAgent) > 0 && userAgent[0] != "" {
-		req.Header.Set("User-Agent", userAgent[0])
+	if len(headers) > 0 {
+		for k, v := range headers[0] {
+			req.Header.Set(k, v)
+		}
 	}
 
 	var resp *http.Response