zenrows-vs-brightdata-benchmark/zenrows.py at main · ZenRows/zenrows-vs-brightdata-benchmark · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
import requests
import time
import csv
import random

import os

YOUR_ZENROWS_API_KEY = os.environ.get("ZENROWS_API_KEY", "")
if not YOUR_ZENROWS_API_KEY:
    raise SystemExit("Set the ZENROWS_API_KEY environment variable before running this script.")


REQUESTS_PER_URL = 100

URLS = [
    {"name": "Amazon", "url": "https://www.amazon.com/ref=nav_logo"},# "https://www.amazon.com/s?k=playstation+gift+card+digital+code&crid=3FXL1NI73ICC&sprefix=playst%2Caps%2C406&ref=nb_sb_ss_p13n-expert-pd-ops-ranker_1_6"}, # https://www.amazon.com/dp/B09B8YWXDF
    {"name": "Glassdoor", "url": "https://www.glassdoor.com/Overview/Working-at-Google-EI_IE9079.11,17.htm"},
    {"name": "Google", "url": "https://www.google.com/search?q=web+scraping+api"},
    {"name": "LinkedIn",   "url": "https://www.linkedin.com/in/satyanadella/"},
    {"name": "Mid-sized e-commerce site behind Cloudflare",   "url": "https://www.ikea.com/"},
    {"name": "Zillow real estate listing", "url": "https://www.zillow.com/homes/for_sale/"},
    {"name": "BBC", "url": "https://www.bbc.com/news/technology"},
]

results = []


def is_valid(html):
    if not html:
        return False

    html_lower = html.lower()

    blocked_signals = [
        "captcha",
        "just a moment",
        "verify you are human",
        "access denied"
    ]

    if any(x in html_lower for x in blocked_signals):
        return False

    return len(html) > 5000

# ----------------------------
# Zenrows
# ----------------------------
print("\n--- Zenrows Tests ---\n")

for target in URLS:
    for i in range(REQUESTS_PER_URL):

        start = time.time()

        try:
            r = requests.get(
                "https://api.zenrows.com/v1/",
                params={
                    "url": target["url"],
                    "apikey": YOUR_ZENROWS_API_KEY,
                    "mode": "auto"
                   # "js_render": "true",
                   # "premium_proxy": "true"
                },
                timeout=60
            )

            html = r.text
            elapsed = round((time.time() - start) * 1000, 2)

            valid = is_valid(html)

            status = r.status_code

        except Exception as e:
            html = ""
            elapsed = 0
            valid = False
            status = f"ERROR: {type(e)}"

        results.append([
            "Zenrows",
            target["name"],
            i + 1,
            status,
            valid,
            elapsed
        ])

        print(f"Zenrows | {target['name']} | #{i+1} | {status} | {valid} | {elapsed}ms")

        time.sleep(0.5)


# ----------------------------
# Save CSV
# ----------------------------
with open("zenrows_benchmark_results.csv", "w", newline="") as f:
    writer = csv.writer(f)
    writer.writerow([
        "platform",
        "target",
        "request_num",
        "status_code",
        "valid",
        "response_time_ms"
    ])
    writer.writerows(results)

print("\n--- DONE: benchmark_results.csv created ---")