Skip to content

Commit cf7a9a1

Browse files
tomayacmax-ostapenko
tomayac
and
max-ostapenko
authored
fix: count sites instead of hostnames in public_hash_list (#357)
* feat: add sync_public_suffix_private task to crawl_complete DAG Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> * refactor: expand public suffix list synchronization to include ICANN domains and exception rules Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> * docs: update documentation to reflect Airflow orchestration for pipelines Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> * fix: count sites instead of hostnames in public_hash_list The >=100 threshold counted distinct hostnames of the resource URL, so a resource used only on many subdomains of one site (for example $city.$vulnerable-group.com) passed the privacy gate. Count distinct sites (eTLD+1 of the embedding page) using the full Public Suffix List, including the private section that BigQuery's NET.PUBLIC_SUFFIX() skips. The private rules come from the urls.public_suffix_list table, which the crawl_complete DAG keeps current. --------- Signed-off-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com>
1 parent 3d9c895 commit cf7a9a1

1 file changed

Lines changed: 98 additions & 13 deletions

File tree

‎definitions/output/performance/public_hash_list.js‎

Lines changed: 98 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,40 @@
11
publish('public_hash_list', {
22
type: 'table',
33
schema: 'performance',
4-
description: `Identifies web resources (scripts, CSS, fonts, WASM) whose SHA-256 body hash appears across >=100 independent origins in the HTTP Archive monthly crawl.
5-
The >=100-origins threshold is the k-anonymity privacy gate: a resource that widespread cannot serve as a cross-site identifier.
6-
Traffic-weighted score: each hash is scored by SUM(100000 / min_rank) across origins, where min_rank is the CrUX popularity bucket of the hosting page (1 000 = top 1K sites -> 100 pts; 1 000 000 = top 1M -> 0.1 pt). This lifts hashes carried by high-traffic pages to the top of the list.
4+
description: `Identifies web resources (scripts, CSS, fonts, WASM) whose SHA-256 body hash appears across >=100 independent sites in the HTTP Archive monthly crawl.
5+
A site is the registrable domain (eTLD+1) of the embedding page, computed with the full Public Suffix List (ICANN and private sections), so city1.example.com and city2.example.com count as the single site example.com, while alice.github.io and bob.github.io count as two sites.
6+
The >=100-sites threshold is the k-anonymity privacy gate: a resource that widespread cannot serve as a cross-site identifier.
7+
Traffic-weighted score: each hash is scored by SUM(100000 / min_rank) across sites, where min_rank is the CrUX popularity bucket of the site's most popular embedding page (1 000 = top 1K sites -> 100 pts; 1 000 000 = top 1M -> 0.1 pt). This lifts hashes carried by high-traffic pages to the top of the list.
78
Results are published as a world-readable Google Sheet: https://docs.google.com/spreadsheets/d/1Cw4wguQ0X4xMqZlTRYlo6OHQaUr7UVYlFZaIQkXK2Jw/edit?usp=sharing
89
The CSV export used by http-archive.js: https://docs.google.com/spreadsheets/d/e/2PACX-1vTOcTespiVHDRLIq16_3GsnnvJmut00x0fzWTLXSWBNya6Go_1kBrGoVJvxb8gEaP_L9FfKmXy3-kF-/pub?output=csv
910
Repo: https://github.com/tomayac/public-hash-list`,
1011
columns: {
1112
body_hash: 'SHA-256 body hash of the resource from WebPageTest payload',
1213
type: 'Simplified type of the resource (script, css, font, wasm)',
13-
num_origins:
14-
'Number of independent origins hosting the resource (privacy-gated threshold of >= 100)',
14+
num_sites:
15+
'Number of independent sites (eTLD+1 of the embedding page) loading the resource (privacy-gated threshold of >= 100)',
1516
traffic_weighted_score:
16-
'Traffic-weighted popularity score (SUM(100000 / min_rank) across origins, where min_rank is the CrUX rank bucket of the hosting page)',
17+
'Traffic-weighted popularity score (SUM(100000 / min_rank) across sites, where min_rank is the CrUX rank bucket of the embedding page)',
1718
sample_url: 'A sample URL where this resource was detected'
1819
},
1920
tags: ['crawl_complete']
2021
})
2122
.query(
2223
(ctx) => `
23-
WITH request_origins AS (
24+
-- Private section of the Public Suffix List. BigQuery's NET.PUBLIC_SUFFIX()
25+
-- only applies the ICANN section, which would merge all *.github.io pages
26+
-- into one site.
27+
WITH private_suffixes AS (
28+
SELECT suffix, is_wildcard
29+
FROM ${ctx.ref('urls', 'public_suffix_list')}
30+
WHERE is_private AND NOT is_exception
31+
),
32+
33+
request_pages AS (
2434
SELECT
2535
SAFE.STRING(payload._body_hash) AS body_hash,
2636
type,
27-
NET.HOST(url) AS origin,
37+
LOWER(NET.HOST(page)) AS page_host,
2838
MIN(rank) AS min_rank,
2939
ANY_VALUE(url) AS sample_url
3040
FROM ${ctx.ref('crawl', 'requests')}
@@ -33,20 +43,95 @@ WITH request_origins AS (
3343
AND SAFE.STRING(payload._body_hash) IS NOT NULL
3444
AND type IN ('script', 'css', 'font', 'wasm')
3545
AND SAFE.STRING(summary.method) = 'GET'
46+
AND NET.HOST(page) IS NOT NULL
3647
GROUP BY
3748
body_hash,
3849
type,
39-
origin
50+
page_host
51+
),
52+
53+
-- Every suffix of every page host, for example a.example.co.uk yields
54+
-- uk, co.uk, example.co.uk, and a.example.co.uk.
55+
host_suffixes AS (
56+
SELECT
57+
host,
58+
ARRAY_LENGTH(SPLIT(host, '.')) AS host_labels,
59+
num_labels,
60+
ARRAY_TO_STRING(
61+
ARRAY(
62+
SELECT label
63+
FROM UNNEST(SPLIT(host, '.')) AS label WITH OFFSET AS i
64+
WHERE i >= ARRAY_LENGTH(SPLIT(host, '.')) - num_labels
65+
ORDER BY i
66+
),
67+
'.'
68+
) AS suffix
69+
FROM (SELECT DISTINCT page_host AS host FROM request_pages)
70+
CROSS JOIN UNNEST(GENERATE_ARRAY(1, ARRAY_LENGTH(SPLIT(host, '.')))) AS num_labels
71+
),
72+
73+
-- Length in labels of each host's longest matching public suffix, ICANN or
74+
-- private. A wildcard rule *.x makes the suffix one label longer than x
75+
-- public. 0 means no rule matched (IP addresses, unknown TLDs).
76+
public_suffix_lengths AS (
77+
SELECT
78+
s.host,
79+
ANY_VALUE(s.host_labels) AS host_labels,
80+
MAX(
81+
CASE
82+
WHEN p.is_wildcard THEN s.num_labels + 1
83+
WHEN p.suffix IS NOT NULL THEN s.num_labels
84+
WHEN s.suffix = NET.PUBLIC_SUFFIX(s.host) THEN s.num_labels
85+
ELSE 0
86+
END
87+
) AS suffix_labels
88+
FROM host_suffixes AS s
89+
LEFT JOIN private_suffixes AS p
90+
ON s.suffix = p.suffix
91+
GROUP BY s.host
92+
),
93+
94+
-- The site is the public suffix plus one label. Hosts without a public
95+
-- suffix, or that are a public suffix themselves, are their own site.
96+
host_sites AS (
97+
SELECT
98+
s.host,
99+
s.suffix AS site
100+
FROM host_suffixes AS s
101+
INNER JOIN public_suffix_lengths AS l
102+
ON s.host = l.host
103+
WHERE
104+
s.num_labels = IF(
105+
l.suffix_labels = 0,
106+
l.host_labels,
107+
LEAST(l.suffix_labels + 1, l.host_labels)
108+
)
109+
),
110+
111+
request_sites AS (
112+
SELECT
113+
r.body_hash,
114+
r.type,
115+
h.site,
116+
MIN(r.min_rank) AS min_rank,
117+
ANY_VALUE(r.sample_url) AS sample_url
118+
FROM request_pages AS r
119+
INNER JOIN host_sites AS h
120+
ON r.page_host = h.host
121+
GROUP BY
122+
r.body_hash,
123+
r.type,
124+
h.site
40125
),
41126
42127
hash_popularity AS (
43128
SELECT
44129
body_hash,
45130
type,
46-
COUNT(DISTINCT origin) AS num_origins,
131+
COUNT(*) AS num_sites,
47132
SUM(100000.0 / min_rank) AS traffic_weighted_score,
48133
ANY_VALUE(sample_url) AS sample_url
49-
FROM request_origins
134+
FROM request_sites
50135
GROUP BY
51136
body_hash,
52137
type
@@ -55,12 +140,12 @@ hash_popularity AS (
55140
SELECT
56141
body_hash,
57142
type,
58-
num_origins,
143+
num_sites,
59144
traffic_weighted_score,
60145
sample_url
61146
FROM hash_popularity
62147
WHERE
63-
num_origins >= 100
148+
num_sites >= 100
64149
ORDER BY
65150
traffic_weighted_score DESC
66151
`

0 commit comments

Comments
 (0)