Skip to content

Commit cec6cfb

Browse files
atoato
committed
Add BotBlockDetector
Detects responses produced by common bot-blocking services and adds a "botblock:service" annotation. This is intended for reporting and troubleshooting but may also be useful as a signal to stop crawling or to skip writing WARC records.
1 parent dea9227 commit cec6cfb

3 files changed

Lines changed: 325 additions & 0 deletions

File tree

‎docs/bean-reference.rst‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -521,6 +521,24 @@ ScrollDownBehavior
521521

522522
.. bean-doc:: org.archive.modules.behaviors.ScrollDownBehavior
523523

524+
Miscellaneous Processors
525+
------------------------
526+
527+
BotBlockDetector
528+
~~~~~~~~~~~~~~~~
529+
530+
.. bean-doc:: org.archive.modules.processor.BotBlockDetector
531+
532+
HashCrawlMapper
533+
~~~~~~~~~~~~~~~
534+
535+
.. bean-doc:: org.archive.crawler.processor.HashCrawlMapper
536+
537+
LexicalCrawlMapper
538+
~~~~~~~~~~~~~~~~~~
539+
540+
.. bean-doc:: org.archive.crawler.processor.LexicalCrawlMapper
541+
524542
Post-Processors
525543
---------------
526544

‎modules/src/main/java/org/archive/modules/processor/BotBlockDetector.java‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
/*
2+
* This file is part of the Heritrix web crawler (crawler.archive.org).
3+
*
4+
* Licensed to the Internet Archive (IA) by one or more individual
5+
* contributors.
6+
*
7+
* The IA licenses this file to You under the Apache License, Version 2.0
8+
* (the "License"); you may not use this file except in compliance with
9+
* the License. You may obtain a copy of the License at
10+
*
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
*
13+
* Unless required by applicable law or agreed to in writing, software
14+
* distributed under the License is distributed on an "AS IS" BASIS,
15+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
* See the License for the specific language governing permissions and
17+
* limitations under the License.
18+
*/
19+
package org.archive.modules.processor;
20+
21+
import java.util.Map;
22+
import java.util.TreeMap;
23+
import java.util.concurrent.ConcurrentHashMap;
24+
import java.util.concurrent.ConcurrentMap;
25+
import java.util.concurrent.atomic.AtomicLong;
26+
27+
import org.archive.modules.CrawlURI;
28+
import org.archive.modules.Processor;
29+
import org.archive.util.JSONUtils;
30+
import org.json.JSONException;
31+
import org.json.JSONObject;
32+
33+
/**
34+
* Detects responses produced by bot-blocking services and adds a "botblock:service" annotation.
35+
* <p>
36+
* Normally added to the fetch chain before the extractors.
37+
*/
38+
public class BotBlockDetector extends Processor {
39+
protected final ConcurrentMap<String, AtomicLong> counts = new ConcurrentHashMap<>();
40+
41+
public Map<String, AtomicLong> getCounts() {
42+
return counts;
43+
}
44+
45+
@Override
46+
protected boolean shouldProcess(CrawlURI curi) {
47+
return curi.isHttpTransaction() &&
48+
curi.getFetchStatus() > 0;
49+
}
50+
51+
@Override
52+
protected void innerProcess(CrawlURI curi) throws InterruptedException {
53+
String detected = detect(curi);
54+
if (detected != null &&
55+
curi.getAnnotations().add("botblock:" + detected)) {
56+
counts.computeIfAbsent(detected, ignored -> new AtomicLong()).incrementAndGet();
57+
}
58+
}
59+
60+
@Override
61+
protected JSONObject toCheckpointJson() throws JSONException {
62+
JSONObject json = super.toCheckpointJson();
63+
json.put("counts", counts);
64+
return json;
65+
}
66+
67+
@Override
68+
protected void fromCheckpointJson(JSONObject json) throws JSONException {
69+
super.fromCheckpointJson(json);
70+
counts.clear();
71+
JSONObject counts = json.optJSONObject("counts");
72+
if (counts != null) JSONUtils.putAllAtomicLongs(this.counts, counts);
73+
}
74+
75+
@Override
76+
public String report() {
77+
return super.report() + " Blocked requests by service: " +
78+
new TreeMap<>(counts) + "\n";
79+
}
80+
81+
protected static String detect(CrawlURI curi) {
82+
if (detectAkamai(curi)) return "akamai";
83+
if (detectAnubis(curi)) return "anubis";
84+
if (detectCloudflare(curi)) return "cloudflare";
85+
if (detectIncapsula(curi)) return "incapsula";
86+
return null;
87+
}
88+
89+
private static boolean detectAkamai(CrawlURI curi) {
90+
return curi.getFetchStatus() == 403 &&
91+
"AkamaiGHost".equals(curi.getHttpResponseHeader("server"));
92+
}
93+
94+
private static boolean detectAnubis(CrawlURI curi) {
95+
if (curi.getFetchStatus() == 307) {
96+
String location = curi.getHttpResponseHeader("location");
97+
return location != null && location.contains("/.within.website/?redir=");
98+
} else if (curi.getFetchStatus() == 200) {
99+
String setCookie = curi.getHttpResponseHeader("set-cookie");
100+
return setCookie != null && setCookie.startsWith("techaro.lol-anubis-") &&
101+
bodyContainsHtml(curi, "<script id=\"anubis_challenge\"");
102+
}
103+
return false;
104+
}
105+
106+
private static boolean detectCloudflare(CrawlURI curi) {
107+
if ("cloudflare".equalsIgnoreCase(curi.getHttpResponseHeader("server"))) {
108+
if ("challenge".equalsIgnoreCase(curi.getHttpResponseHeader("cf-mitigated"))) {
109+
return true;
110+
}
111+
return curi.getFetchStatus() == 403 &&
112+
bodyContainsHtml(curi, "<h1 data-translate=\"block_headline\">Sorry, you have been blocked</h1>");
113+
}
114+
return false;
115+
}
116+
117+
private static boolean detectIncapsula(CrawlURI curi) {
118+
return curi.getFetchStatus() == 404 &&
119+
curi.getHttpResponseHeader("x-iinfo") != null &&
120+
bodyContainsHtml(curi, "Incapsula incident ID:");
121+
}
122+
123+
private static boolean bodyContainsHtml(CrawlURI curi, String string) {
124+
return hasHtmlContentType(curi) &&
125+
curi.getRecorder() != null &&
126+
curi.getRecorder().getContentReplayPrefixString(8000).contains(string);
127+
}
128+
129+
private static boolean hasHtmlContentType(CrawlURI curi) {
130+
String type = curi.getContentType();
131+
return type == null ||
132+
matchesMediaType(type, "text/html") ||
133+
matchesMediaType(type, "application/xhtml+xml");
134+
}
135+
136+
private static boolean matchesMediaType(String value, String expected) {
137+
if (!value.regionMatches(true, 0, expected, 0, expected.length())) {
138+
return false;
139+
}
140+
int i = expected.length();
141+
while (i < value.length() && Character.isWhitespace(value.charAt(i))) {
142+
i++;
143+
}
144+
return i == value.length() || value.charAt(i) == ';';
145+
}
146+
}

‎modules/src/test/java/org/archive/modules/processor/BotBlockDetectorTest.java‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
/*
2+
* This file is part of the Heritrix web crawler (crawler.archive.org).
3+
*
4+
* Licensed to the Internet Archive (IA) by one or more individual
5+
* contributors.
6+
*
7+
* The IA licenses this file to You under the Apache License, Version 2.0
8+
* (the "License"); you may not use this file except in compliance with
9+
* the License. You may obtain a copy of the License at
10+
*
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
*
13+
* Unless required by applicable law or agreed to in writing, software
14+
* distributed under the License is distributed on an "AS IS" BASIS,
15+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
* See the License for the specific language governing permissions and
17+
* limitations under the License.
18+
*/
19+
package org.archive.modules.processor;
20+
21+
import java.io.ByteArrayInputStream;
22+
import java.nio.charset.StandardCharsets;
23+
import java.nio.file.Path;
24+
import java.util.Set;
25+
26+
import org.archive.modules.CrawlURI;
27+
import org.archive.net.UURIFactory;
28+
import org.archive.util.Recorder;
29+
import org.junit.jupiter.api.Test;
30+
import org.junit.jupiter.api.io.TempDir;
31+
32+
import static org.junit.jupiter.api.Assertions.*;
33+
34+
class BotBlockDetectorTest {
35+
36+
@TempDir
37+
Path tempDir;
38+
39+
@Test
40+
void detectsAkamaiBlock() throws Exception {
41+
assertBlocked(akamaiBlock(), "akamai");
42+
}
43+
44+
@Test
45+
void detectsAnubisRedirectChallenge() throws Exception {
46+
CrawlURI curi = curi(307);
47+
curi.putHttpResponseHeader("Location",
48+
"/.within.website/?redir=https%3A%2F%2Fexample.com%2F");
49+
50+
assertBlocked(curi, "anubis");
51+
}
52+
53+
@Test
54+
void detectsAnubisCookieChallenge() throws Exception {
55+
CrawlURI curi = curi(200);
56+
curi.putHttpResponseHeader("Set-Cookie",
57+
"techaro.lol-anubis-auth=token; Path=/");
58+
recordResponse(curi, "<script id=\"anubis_challenge\"></script>");
59+
60+
assertBlocked(curi, "anubis");
61+
}
62+
63+
@Test
64+
void ordinaryResponseDoesNotMatch() throws Exception {
65+
CrawlURI curi = curi(200);
66+
67+
assertNotBlocked(curi);
68+
}
69+
70+
@Test
71+
void detectsCloudflareChallenge() throws Exception {
72+
CrawlURI curi = curi(403);
73+
curi.putHttpResponseHeader("Server", "cloudflare");
74+
curi.putHttpResponseHeader("CF-Mitigated", "challenge");
75+
76+
assertBlocked(curi, "cloudflare");
77+
}
78+
79+
@Test
80+
void detectsCloudflareBlock() throws Exception {
81+
assertBlocked(cloudflareBlock(), "cloudflare");
82+
}
83+
84+
@Test
85+
void detectsIncapsulaChallenge() throws Exception {
86+
CrawlURI curi = curi(404);
87+
curi.putHttpResponseHeader("x-iinfo", "foo");
88+
recordResponse(curi,
89+
"Request unsuccessful. Incapsula incident ID: 123456");
90+
91+
assertBlocked(curi, "incapsula");
92+
}
93+
94+
@Test
95+
void unrelated403ResponseDoesNotMatch() throws Exception {
96+
CrawlURI curi = curi(403);
97+
curi.putHttpResponseHeader("Server", "example");
98+
99+
assertNotBlocked(curi);
100+
}
101+
102+
@Test
103+
void checkpointRoundTripPreservesBlockedRequestCounts() throws Exception {
104+
BotBlockDetector detector = new BotBlockDetector();
105+
detector.process(akamaiBlock());
106+
detector.process(cloudflareBlock());
107+
108+
BotBlockDetector restored = new BotBlockDetector();
109+
restored.fromCheckpointJson(detector.toCheckpointJson());
110+
111+
assertEquals(1, restored.getCounts().get("akamai").get());
112+
assertEquals(1, restored.getCounts().get("cloudflare").get());
113+
assertEquals(2, restored.getURICount());
114+
}
115+
116+
private static void assertBlocked(CrawlURI curi, String service)
117+
throws InterruptedException {
118+
new BotBlockDetector().process(curi);
119+
assertEquals(Set.of("botblock:" + service), curi.getAnnotations());
120+
}
121+
122+
private static void assertNotBlocked(CrawlURI curi)
123+
throws InterruptedException {
124+
new BotBlockDetector().process(curi);
125+
assertTrue(curi.getAnnotations().isEmpty());
126+
}
127+
128+
private static CrawlURI akamaiBlock() throws Exception {
129+
CrawlURI curi = curi(403);
130+
curi.putHttpResponseHeader("Server", "AkamaiGHost");
131+
return curi;
132+
}
133+
134+
private CrawlURI cloudflareBlock() throws Exception {
135+
CrawlURI curi = curi(403);
136+
curi.putHttpResponseHeader("Server", "cloudflare");
137+
recordResponse(curi, "<h1 data-translate=\"block_headline\">Sorry, you have been blocked</h1>");
138+
return curi;
139+
}
140+
141+
private static CrawlURI curi(int status) throws Exception {
142+
CrawlURI curi = new CrawlURI(UURIFactory.getInstance("https://example.com/"));
143+
curi.setFetchStatus(status);
144+
curi.setFetchType(CrawlURI.FetchType.HTTP_GET);
145+
curi.setContentType("text/html");
146+
return curi;
147+
}
148+
149+
private void recordResponse(CrawlURI curi, String body) throws Exception {
150+
byte[] response = ("HTTP/1.1 " + curi.getFetchStatus() + " Test\r\n"
151+
+ "Content-Type: text/html\r\n"
152+
+ "Content-Length: " + body.getBytes(StandardCharsets.UTF_8).length + "\r\n"
153+
+ "\r\n"
154+
+ body).getBytes(StandardCharsets.UTF_8);
155+
Recorder recorder = new Recorder(tempDir.toFile(), "bot-block-detector");
156+
curi.setRecorder(recorder);
157+
recorder.inputWrap(new ByteArrayInputStream(response));
158+
recorder.getRecordedInput().readFully();
159+
recorder.close();
160+
}
161+
}

0 commit comments

Comments
 (0)