Skip to content

Commit 8f10abd

Browse files
author
stlc-bot
committed
feat(scrape): add Tripadvisor reviews parser (#2508)
Stainless-Generated-From: 4262794e7f98f160ef520bf4bb77c09ce3a1615a
1 parent 6495548 commit 8f10abd

12 files changed

Lines changed: 378 additions & 165 deletions

‎lib/context_dev/models/batch_get_results_response.rb‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,9 @@ class Result < ContextDev::Internal::Type::BaseModel
193193
# data.users, yelp-search returns up to 20 ranked organic businesses in
194194
# data.businesses, plus total and pagination, from a Yelp search URL; yelp-reviews
195195
# returns up to 20 latest public reviews in data.reviews, with business details
196-
# and next_offset, from a Yelp business URL; emails, phone-numbers, and links
196+
# and next_offset, from a Yelp business URL; tripadvisor-reviews returns 20
197+
# reviews in data.reviews, with data.location and next_offset, from a Tripadvisor
198+
# hotel, restaurant, or attraction review URL; emails, phone-numbers, and links
197199
# return data.emails, data.phone_numbers, and data.links from the rendered page;
198200
# the store, Maps, G2, Instagram, and TikTok parsers read their data directly,
199201
# independently of DOM filters and other outputs.
@@ -1033,7 +1035,9 @@ class Parsed < ContextDev::Internal::Type::BaseModel
10331035
# data.users, yelp-search returns up to 20 ranked organic businesses in
10341036
# data.businesses, plus total and pagination, from a Yelp search URL; yelp-reviews
10351037
# returns up to 20 latest public reviews in data.reviews, with business details
1036-
# and next_offset, from a Yelp business URL; emails, phone-numbers, and links
1038+
# and next_offset, from a Yelp business URL; tripadvisor-reviews returns 20
1039+
# reviews in data.reviews, with data.location and next_offset, from a Tripadvisor
1040+
# hotel, restaurant, or attraction review URL; emails, phone-numbers, and links
10371041
# return data.emails, data.phone_numbers, and data.links from the rendered page;
10381042
# the store, Maps, G2, Instagram, and TikTok parsers read their data directly,
10391043
# independently of DOM filters and other outputs.
@@ -1081,6 +1085,7 @@ module Type
10811085
REDDIT_USER = :"reddit-user"
10821086
YELP_SEARCH = :"yelp-search"
10831087
YELP_REVIEWS = :"yelp-reviews"
1088+
TRIPADVISOR_REVIEWS = :"tripadvisor-reviews"
10841089
EMAILS = :emails
10851090
PHONE_NUMBERS = :"phone-numbers"
10861091
LINKS = :links
@@ -1607,7 +1612,9 @@ class Result < ContextDev::Internal::Type::BaseModel
16071612
# data.users, yelp-search returns up to 20 ranked organic businesses in
16081613
# data.businesses, plus total and pagination, from a Yelp search URL; yelp-reviews
16091614
# returns up to 20 latest public reviews in data.reviews, with business details
1610-
# and next_offset, from a Yelp business URL; emails, phone-numbers, and links
1615+
# and next_offset, from a Yelp business URL; tripadvisor-reviews returns 20
1616+
# reviews in data.reviews, with data.location and next_offset, from a Tripadvisor
1617+
# hotel, restaurant, or attraction review URL; emails, phone-numbers, and links
16111618
# return data.emails, data.phone_numbers, and data.links from the rendered page;
16121619
# the store, Maps, G2, Instagram, and TikTok parsers read their data directly,
16131620
# independently of DOM filters and other outputs.
@@ -2449,7 +2456,9 @@ class Parsed < ContextDev::Internal::Type::BaseModel
24492456
# data.users, yelp-search returns up to 20 ranked organic businesses in
24502457
# data.businesses, plus total and pagination, from a Yelp search URL; yelp-reviews
24512458
# returns up to 20 latest public reviews in data.reviews, with business details
2452-
# and next_offset, from a Yelp business URL; emails, phone-numbers, and links
2459+
# and next_offset, from a Yelp business URL; tripadvisor-reviews returns 20
2460+
# reviews in data.reviews, with data.location and next_offset, from a Tripadvisor
2461+
# hotel, restaurant, or attraction review URL; emails, phone-numbers, and links
24532462
# return data.emails, data.phone_numbers, and data.links from the rendered page;
24542463
# the store, Maps, G2, Instagram, and TikTok parsers read their data directly,
24552464
# independently of DOM filters and other outputs.
@@ -2497,6 +2506,7 @@ module Type
24972506
REDDIT_USER = :"reddit-user"
24982507
YELP_SEARCH = :"yelp-search"
24992508
YELP_REVIEWS = :"yelp-reviews"
2509+
TRIPADVISOR_REVIEWS = :"tripadvisor-reviews"
25002510
EMAILS = :emails
25012511
PHONE_NUMBERS = :"phone-numbers"
25022512
LINKS = :links

‎lib/context_dev/models/batch_submit_params.rb‎

Lines changed: 78 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1731,16 +1731,25 @@ class PrebuiltParser < ContextDev::Internal::Type::BaseModel
17311731
# pagination.next_offset. Each review has rating, full text, updated_at, and
17321732
# author (id is the Yelp user id from the reviewer profile URL, url links to that
17331733
# profile, plus name, location, and review_count); owner replies and photos are
1734-
# not included. emails accepts any page and returns its email addresses as
1735-
# data.emails ([{ email }]), read from mailto links, Cloudflare-protected
1736-
# addresses, and visible text. phone-numbers accepts any page and returns its
1737-
# valid phone numbers in E.164 as data.phone_numbers ([{ phone }]), read from tel
1738-
# links and visible text; numbers without a country code are read for the country
1739-
# of the site TLD, defaulting to US. links accepts any page and returns its unique
1740-
# http(s) links as data.links ([{ url, text }]), with absolute URLs; text falls
1741-
# back to aria-label or image alt and is null when a link has none. The emails
1742-
# parser reads the rendered page and respects content filters; the other prebuilt
1743-
# parsers read their data independently of DOM filters.
1734+
# not included. tripadvisor-reviews accepts a Tripadvisor hotel, restaurant, or
1735+
# attraction review URL (tripadvisor.com/Hotel_Review-g{geo}-d{id}-Reviews-….html,
1736+
# on any Tripadvisor country domain), reads -or{N}- in the path as the offset, and
1737+
# returns 20 reviews in data.reviews in the order Tripadvisor shows them, with
1738+
# data.location (id, name, type, url, review_count) and pagination.next_offset.
1739+
# Each review has id, url, title, full text, rating, language, published_at,
1740+
# visit_month, helpful_votes, subratings, photo_urls, owner_response (text,
1741+
# published_at, author_name), and author (id, username, name, url,
1742+
# contribution_count); an unknown location returns no reviews. emails accepts any
1743+
# page and returns its email addresses as data.emails ([{ email }]), read from
1744+
# mailto links, Cloudflare-protected addresses, and visible text. phone-numbers
1745+
# accepts any page and returns its valid phone numbers in E.164 as
1746+
# data.phone_numbers ([{ phone }]), read from tel links and visible text; numbers
1747+
# without a country code are read for the country of the site TLD, defaulting to
1748+
# US. links accepts any page and returns its unique http(s) links as data.links
1749+
# ([{ url, text }]), with absolute URLs; text falls back to aria-label or image
1750+
# alt and is null when a link has none. The emails parser reads the rendered page
1751+
# and respects content filters; the other prebuilt parsers read their data
1752+
# independently of DOM filters.
17441753
#
17451754
# @return [Symbol, ContextDev::Models::BatchSubmitParams::Input::Scrape::Data::BatchUnifiedScrapeData::ParseParams::PrebuiltParser::Parser]
17461755
required :parser,
@@ -1900,16 +1909,25 @@ class PrebuiltParser < ContextDev::Internal::Type::BaseModel
19001909
# pagination.next_offset. Each review has rating, full text, updated_at, and
19011910
# author (id is the Yelp user id from the reviewer profile URL, url links to that
19021911
# profile, plus name, location, and review_count); owner replies and photos are
1903-
# not included. emails accepts any page and returns its email addresses as
1904-
# data.emails ([{ email }]), read from mailto links, Cloudflare-protected
1905-
# addresses, and visible text. phone-numbers accepts any page and returns its
1906-
# valid phone numbers in E.164 as data.phone_numbers ([{ phone }]), read from tel
1907-
# links and visible text; numbers without a country code are read for the country
1908-
# of the site TLD, defaulting to US. links accepts any page and returns its unique
1909-
# http(s) links as data.links ([{ url, text }]), with absolute URLs; text falls
1910-
# back to aria-label or image alt and is null when a link has none. The emails
1911-
# parser reads the rendered page and respects content filters; the other prebuilt
1912-
# parsers read their data independently of DOM filters.
1912+
# not included. tripadvisor-reviews accepts a Tripadvisor hotel, restaurant, or
1913+
# attraction review URL (tripadvisor.com/Hotel_Review-g{geo}-d{id}-Reviews-….html,
1914+
# on any Tripadvisor country domain), reads -or{N}- in the path as the offset, and
1915+
# returns 20 reviews in data.reviews in the order Tripadvisor shows them, with
1916+
# data.location (id, name, type, url, review_count) and pagination.next_offset.
1917+
# Each review has id, url, title, full text, rating, language, published_at,
1918+
# visit_month, helpful_votes, subratings, photo_urls, owner_response (text,
1919+
# published_at, author_name), and author (id, username, name, url,
1920+
# contribution_count); an unknown location returns no reviews. emails accepts any
1921+
# page and returns its email addresses as data.emails ([{ email }]), read from
1922+
# mailto links, Cloudflare-protected addresses, and visible text. phone-numbers
1923+
# accepts any page and returns its valid phone numbers in E.164 as
1924+
# data.phone_numbers ([{ phone }]), read from tel links and visible text; numbers
1925+
# without a country code are read for the country of the site TLD, defaulting to
1926+
# US. links accepts any page and returns its unique http(s) links as data.links
1927+
# ([{ url, text }]), with absolute URLs; text falls back to aria-label or image
1928+
# alt and is null when a link has none. The emails parser reads the rendered page
1929+
# and respects content filters; the other prebuilt parsers read their data
1930+
# independently of DOM filters.
19131931
#
19141932
# @see ContextDev::Models::BatchSubmitParams::Input::Scrape::Data::BatchUnifiedScrapeData::ParseParams::PrebuiltParser#parser
19151933
module Parser
@@ -1938,6 +1956,7 @@ module Parser
19381956
REDDIT_USER = :"reddit-user"
19391957
YELP_SEARCH = :"yelp-search"
19401958
YELP_REVIEWS = :"yelp-reviews"
1959+
TRIPADVISOR_REVIEWS = :"tripadvisor-reviews"
19411960
EMAILS = :emails
19421961
PHONE_NUMBERS = :"phone-numbers"
19431962
LINKS = :links
@@ -5162,16 +5181,25 @@ class PrebuiltParser < ContextDev::Internal::Type::BaseModel
51625181
# pagination.next_offset. Each review has rating, full text, updated_at, and
51635182
# author (id is the Yelp user id from the reviewer profile URL, url links to that
51645183
# profile, plus name, location, and review_count); owner replies and photos are
5165-
# not included. emails accepts any page and returns its email addresses as
5166-
# data.emails ([{ email }]), read from mailto links, Cloudflare-protected
5167-
# addresses, and visible text. phone-numbers accepts any page and returns its
5168-
# valid phone numbers in E.164 as data.phone_numbers ([{ phone }]), read from tel
5169-
# links and visible text; numbers without a country code are read for the country
5170-
# of the site TLD, defaulting to US. links accepts any page and returns its unique
5171-
# http(s) links as data.links ([{ url, text }]), with absolute URLs; text falls
5172-
# back to aria-label or image alt and is null when a link has none. The emails
5173-
# parser reads the rendered page and respects content filters; the other prebuilt
5174-
# parsers read their data independently of DOM filters.
5184+
# not included. tripadvisor-reviews accepts a Tripadvisor hotel, restaurant, or
5185+
# attraction review URL (tripadvisor.com/Hotel_Review-g{geo}-d{id}-Reviews-….html,
5186+
# on any Tripadvisor country domain), reads -or{N}- in the path as the offset, and
5187+
# returns 20 reviews in data.reviews in the order Tripadvisor shows them, with
5188+
# data.location (id, name, type, url, review_count) and pagination.next_offset.
5189+
# Each review has id, url, title, full text, rating, language, published_at,
5190+
# visit_month, helpful_votes, subratings, photo_urls, owner_response (text,
5191+
# published_at, author_name), and author (id, username, name, url,
5192+
# contribution_count); an unknown location returns no reviews. emails accepts any
5193+
# page and returns its email addresses as data.emails ([{ email }]), read from
5194+
# mailto links, Cloudflare-protected addresses, and visible text. phone-numbers
5195+
# accepts any page and returns its valid phone numbers in E.164 as
5196+
# data.phone_numbers ([{ phone }]), read from tel links and visible text; numbers
5197+
# without a country code are read for the country of the site TLD, defaulting to
5198+
# US. links accepts any page and returns its unique http(s) links as data.links
5199+
# ([{ url, text }]), with absolute URLs; text falls back to aria-label or image
5200+
# alt and is null when a link has none. The emails parser reads the rendered page
5201+
# and respects content filters; the other prebuilt parsers read their data
5202+
# independently of DOM filters.
51755203
#
51765204
# @return [Symbol, ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::BatchUnifiedCrawlData::ParseParams::PrebuiltParser::Parser]
51775205
required :parser,
@@ -5331,16 +5359,25 @@ class PrebuiltParser < ContextDev::Internal::Type::BaseModel
53315359
# pagination.next_offset. Each review has rating, full text, updated_at, and
53325360
# author (id is the Yelp user id from the reviewer profile URL, url links to that
53335361
# profile, plus name, location, and review_count); owner replies and photos are
5334-
# not included. emails accepts any page and returns its email addresses as
5335-
# data.emails ([{ email }]), read from mailto links, Cloudflare-protected
5336-
# addresses, and visible text. phone-numbers accepts any page and returns its
5337-
# valid phone numbers in E.164 as data.phone_numbers ([{ phone }]), read from tel
5338-
# links and visible text; numbers without a country code are read for the country
5339-
# of the site TLD, defaulting to US. links accepts any page and returns its unique
5340-
# http(s) links as data.links ([{ url, text }]), with absolute URLs; text falls
5341-
# back to aria-label or image alt and is null when a link has none. The emails
5342-
# parser reads the rendered page and respects content filters; the other prebuilt
5343-
# parsers read their data independently of DOM filters.
5362+
# not included. tripadvisor-reviews accepts a Tripadvisor hotel, restaurant, or
5363+
# attraction review URL (tripadvisor.com/Hotel_Review-g{geo}-d{id}-Reviews-….html,
5364+
# on any Tripadvisor country domain), reads -or{N}- in the path as the offset, and
5365+
# returns 20 reviews in data.reviews in the order Tripadvisor shows them, with
5366+
# data.location (id, name, type, url, review_count) and pagination.next_offset.
5367+
# Each review has id, url, title, full text, rating, language, published_at,
5368+
# visit_month, helpful_votes, subratings, photo_urls, owner_response (text,
5369+
# published_at, author_name), and author (id, username, name, url,
5370+
# contribution_count); an unknown location returns no reviews. emails accepts any
5371+
# page and returns its email addresses as data.emails ([{ email }]), read from
5372+
# mailto links, Cloudflare-protected addresses, and visible text. phone-numbers
5373+
# accepts any page and returns its valid phone numbers in E.164 as
5374+
# data.phone_numbers ([{ phone }]), read from tel links and visible text; numbers
5375+
# without a country code are read for the country of the site TLD, defaulting to
5376+
# US. links accepts any page and returns its unique http(s) links as data.links
5377+
# ([{ url, text }]), with absolute URLs; text falls back to aria-label or image
5378+
# alt and is null when a link has none. The emails parser reads the rendered page
5379+
# and respects content filters; the other prebuilt parsers read their data
5380+
# independently of DOM filters.
53445381
#
53455382
# @see ContextDev::Models::BatchSubmitParams::Input::Crawl::Data::BatchUnifiedCrawlData::ParseParams::PrebuiltParser#parser
53465383
module Parser
@@ -5369,6 +5406,7 @@ module Parser
53695406
REDDIT_USER = :"reddit-user"
53705407
YELP_SEARCH = :"yelp-search"
53715408
YELP_REVIEWS = :"yelp-reviews"
5409+
TRIPADVISOR_REVIEWS = :"tripadvisor-reviews"
53725410
EMAILS = :emails
53735411
PHONE_NUMBERS = :"phone-numbers"
53745412
LINKS = :links

0 commit comments

Comments
 (0)