Add more bot IP sources

Author: alopatindev

alpine/alpine-root/etc/local.d/01-iptables-extra.start.example

@@ -76,15 +76,15 @@ done
   done
 } &
 
-# disallow traffic from Google Cloud
+# disallow traffic from bots
 {
-  [ ! -f /var/tmp/google-cloud-ipv4.txt ] || [ ! -f /var/tmp/google-cloud-ipv6.txt ] && update-google-ips.sh || :
+  [ ! -f /var/tmp/google-googlebot-ipv4.txt ] || [ ! -f /var/tmp/google-googlebot-ipv6.txt ] || [ ! -f /var/tmp/google-cloud-ipv4.txt ] || [ ! -f /var/tmp/google-cloud-ipv6.txt ] && update-google-ips.sh || :
 
-  for i in $(cat /var/tmp/google-cloud-ipv4.txt) ; do
+  for i in $(cat /var/tmp/google-googlebot-ipv4.txt /var/tmp/google-cloud-ipv4.txt /var/tmp/bots-*-ipv4.txt) ; do
     iptables -A INPUT -s "$i" -j DROP
   done
 
-  for i in $(cat /var/tmp/google-cloud-ipv6.txt) ; do
+  for i in $(cat /var/tmp/google-googlebot-ipv6.txt /var/tmp/google-cloud-ipv6.txt /var/tmp/bots-*-ipv6.txt) ; do
     ip6tables -A INPUT -s "$i" -j DROP
   done
 } &

alpine/alpine-root/etc/periodic/hourly/reload-ips.sh

@@ -5,9 +5,10 @@ reload_rules=0
 update-banlist-dns.sh &
 
 update-allowlist-ips.sh && logger 'updated allowlist IPs' && reload_rules=1
+update-bot-ips.sh && logger 'updated bot IPs' && reload_rules=1
+update-cloudflare-ips.sh && logger 'updated Cloudflare IPs' && reload_rules=1
 update-google-ips.sh && logger 'updated Google IPs' && reload_rules=1
-update-cloudflare-ips.sh && logger 'received new Cloudflare IPs' && reload_rules=1
 
-[ "$reload_rules" = 1 ] && /etc/init.d/local restart | logger
+[ "$reload_rules" = 1 ] && /etc/init.d/local --verbose restart | logger
 
 wait

alpine/alpine-root/usr/local/sbin/update-bot-ips.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+PATH_PREFIX="/var/tmp/bots"
+PREFIX="https://raw.githubusercontent.com/lord-alfred/ipranges/main"
+IPV4=( amazon bing duckassistbot duckduckbot facebook microsoft openai perplexity pingdom statuscake twitter )
+IPV6=( amazon facebook microsoft pingdom twitter )
+
+path() {
+  version="$1"
+  target="$2"
+  echo "${PATH_PREFIX}-${target}-ipv${version}.txt"
+}
+
+load() {
+  for target in "${IPV4[@]}" ; do
+    cat $(path 4 "${target}") 2>>/dev/null
+  done
+
+  for target in "${IPV6[@]}" ; do
+    cat $(path 6 "${target}") 2>>/dev/null
+  done
+}
+
+sum() {
+  load | sha256sum
+}
+
+get() {
+  url="$1"
+  sudo -u nobody wget --timeout=30 --tries=10 -qO - "${url}"
+  echo
+}
+
+write() {
+  version="$1"
+  cidr="$2"
+  target="$3"
+  output=$(path "${version}" "${target}")
+  temp="${output}.tmp"
+  grepcidr -e "${cidr}" | \
+    sort -u > "${temp}"
+  [ "$(stat --format='%s' ${temp})" -gt 1 ] && mv "${temp}" "${output}"
+}
+
+download() {
+  for target in "${IPV4[@]}" ; do
+    get "${PREFIX}/${target}/ipv4_merged.txt" | write 4 '0.0.0.0/0' "${target}"
+  done
+
+  for target in "${IPV6[@]}" ; do
+    get "${PREFIX}/${target}/ipv6_merged.txt" | write 6 '::/0' "${target}"
+  done
+}
+
+before=$(sum)
+download
+after=$(sum)
+[ "${before}" != "${after}" ] && exit 0 || exit 1

alpine/alpine-root/usr/local/sbin/update-cloudflare-ips.sh

@@ -14,12 +14,20 @@ sum() {
   load | sha256sum
 }
 
+get() {
+  url="$1"
+  sudo -u nobody wget --timeout=30 --tries=10 -qO - "${url}"
+  echo
+}
+
 download() {
   version="$1"
   cidr="$2"
   output="${PATH_PREFIX}${version}.txt"
   temp="${output}.tmp"
-  sudo -u nobody wget --timeout=30 --tries=10 -qO - "https://www.cloudflare.com/ips-v${version}" | grepcidr -e "${cidr}" | sort -u > "${temp}"
+  ( get "https://www.cloudflare.com/ips-v${version}" ; get "https://raw.githubusercontent.com/lord-alfred/ipranges/main/cloudflare/ipv${version}_merged.txt" ) | \
+    grepcidr -e "${cidr}" | \
+    sort -u > "${temp}"
   [ "$(stat --format='%s' ${temp})" -gt 1 ] && mv "${temp}" "${output}"
 }
 

alpine/alpine-root/usr/local/sbin/update-google-ips.sh

@@ -3,13 +3,15 @@
 # https://cloud.google.com/compute/docs/faq#find_ip_range
 
 PATH_PREFIX="/var/tmp/google"
+PREFIX="https://raw.githubusercontent.com/lord-alfred/ipranges/main"
 
-load() {
-  cat ${PATH_PREFIX}*ipv{4,6}.txt 2>>/dev/null
+sum() {
+  cat ${PATH_PREFIX}*ipv{4,6}.txt 2>>/dev/null | sha256sum
 }
 
-sum() {
-  load | sha256sum
+parse() {
+  version="$1"
+  jq --raw-output --monochrome-output ".prefixes[] | .ipv${version}Prefix" | grep -v 'null'
 }
 
 write() {
@@ -18,23 +20,31 @@ write() {
   target="$3"
   output="${PATH_PREFIX}-${target}-ipv${version}.txt"
   temp="${output}.tmp"
-  cat | \
-    jq --raw-output --monochrome-output ".prefixes[] | .ipv${version}Prefix" | \
-    grep -v 'null' | \
-    grepcidr -e "${cidr}" | \
-    sort -u > "${temp}"
+  grepcidr -e "${cidr}" | sort -u > "${temp}"
   [ "$(stat --format='%s' ${temp})" -gt 1 ] && mv "${temp}" "${output}"
 }
 
+get() {
+  url="$1"
+  sudo -u nobody wget --timeout=30 --tries=10 -qO - "${url}"
+  echo
+}
+
 download() {
   target="$1"
-  ips=$(sudo -u nobody wget --timeout=30 --tries=10 -qO - "https://www.gstatic.com/ipranges/${target}.json")
-  write 4 '0.0.0.0/0' "${target}" <<< "${ips}"
-  write 6 '::/0' "${target}" <<< "${ips}"
+  ips=$(get "https://www.gstatic.com/ipranges/${target}.json")
+  parse 4 <<< "${ips}" | write 4 '0.0.0.0/0' "${target}"
+  parse 6 <<< "${ips}" | write 6 '::/0' "${target}"
+}
+
+download_googlebot() {
+  get "${PREFIX}/googlebot/ipv4_merged.txt" | write 4 '0.0.0.0/0' 'googlebot'
+  get "${PREFIX}/googlebot/ipv6_merged.txt" | write 6 '::/0' 'googlebot'
 }
 
 before=$(sum)
 download 'cloud'
 download 'goog'
+download_googlebot
 after=$(sum)
 [ "${before}" != "${after}" ] && exit 0 || exit 1