[feature] adblock filter and improve perfomance

Author: batchar2

CMakeLists.txt

@@ -58,6 +58,12 @@ set(FPTN_DEFAULT_BLACKLIST_DOMAINS
 add_compile_definitions(FPTN_CLIENT_DEFAULT_BLACKLIST_DOMAINS=\"${FPTN_DEFAULT_BLACKLIST_DOMAINS}\")
 # server
 add_compile_definitions(FPTN_SERVER_DEFAULT_BLACKLIST_DOMAINS=\"${FPTN_DEFAULT_BLACKLIST_DOMAINS}\")
+set(FPTN_SERVER_DEFAULT_DOMAIN_BLACKLIST_URLS
+    "https://raw.githubusercontent.com/fptn-project/fptn/refs/heads/master/deploy/domain_blacklist/russia.txt")
+add_compile_definitions(FPTN_SERVER_DEFAULT_DOMAIN_BLACKLIST_URLS=\"${FPTN_SERVER_DEFAULT_DOMAIN_BLACKLIST_URLS}\")
+set(FPTN_SERVER_DEFAULT_ADS_BLOCKLIST_URLS
+    "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/ultimate-onlydomains.txt,https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts")
+add_compile_definitions(FPTN_SERVER_DEFAULT_ADS_BLOCKLIST_URLS=\"${FPTN_SERVER_DEFAULT_ADS_BLOCKLIST_URLS}\")
 add_compile_definitions(FPTN_SERVER_DEFAULT_ADDRESS_IP6=\"fc00:1::1\")
 add_compile_definitions(FPTN_SERVER_DEFAULT_NET_ADDRESS_IP6=\"fc00:1::\")
 add_compile_definitions(FPTN_SERVER_DEFAULT_ADDRESS_IP4=\"172.20.0.1\")

deploy/docker/scripts/start-fptn.sh

@@ -3,22 +3,6 @@
 export OUT_NETWORK_INTERFACE=$(ip -o -4 route show to default | awk '{print $5}')
 echo "[FPTN] Using network interface: $OUT_NETWORK_INTERFACE"
 
-# Download the domain blacklist before starting the server.
-BLACKLIST_FILE=/tmp/fptn-blacklist.txt
-BLACKLIST_ARG=""
-rm -f "${BLACKLIST_FILE}"
-if [ -n "${BLACKLIST_URL}" ]; then
-    echo "[FPTN] Downloading domain blacklist from: ${BLACKLIST_URL}"
-    if wget -4 -q --timeout=10 --tries=2 -O "${BLACKLIST_FILE}.tmp" "${BLACKLIST_URL}"; then
-        mv "${BLACKLIST_FILE}.tmp" "${BLACKLIST_FILE}"
-        BLACKLIST_ARG="--domain-blacklist-file=${BLACKLIST_FILE}"
-        echo "[FPTN] Domain blacklist saved to ${BLACKLIST_FILE}"
-    else
-        rm -f "${BLACKLIST_FILE}.tmp"
-        echo "[FPTN] WARNING: failed to download blacklist; using built-in list only"
-    fi
-fi
-
 exec /usr/local/bin/fptn-server \
     --server-key=/etc/fptn/server.key \
     --server-crt=/etc/fptn/server.crt \
@@ -28,13 +12,17 @@ exec /usr/local/bin/fptn-server \
     --default-proxy-domain="${DEFAULT_PROXY_DOMAIN}" \
     --allowed-sni-list="${ALLOWED_SNI_LIST}" \
     --tun-interface-name=fptn0 \
-    --disable-torrent-filter="$DISABLE_TORRENT_FILTER" \
-    --disable-spam-filter="$DISABLE_SPAM_FILTER" \
+    --enable-domain-blacklist-filter="$ENABLE_DOMAIN_BLACKLIST_FILTER" \
+    --domain-blacklist-urls="$DOMAIN_BLACKLIST_URLS" \
+    --enable-ads-filter="$ENABLE_ADS_FILTER" \
+    --ads-blocklist-urls="$ADS_BLOCKLIST_URLS" \
+    --data-dir="$DATA_DIR" \
+    --enable-torrent-filter="$ENABLE_TORRENT_FILTER" \
+    --enable-spam-filter="$ENABLE_SPAM_FILTER" \
     --prometheus-access-key="$PROMETHEUS_SECRET_ACCESS_KEY" \
     --use-remote-server-auth="$USE_REMOTE_SERVER_AUTH" \
     --remote-server-auth-host="$REMOTE_SERVER_AUTH_HOST" \
     --remote-server-auth-port="$REMOTE_SERVER_AUTH_PORT" \
     --max-active-sessions-per-user="$MAX_ACTIVE_SESSIONS_PER_USER" \
     --server-external-ips="${SERVER_EXTERNAL_IPS}" \
-    --mtu-size="${MTU_SIZE}" \
-    ${BLACKLIST_ARG}
+    --mtu-size="${MTU_SIZE}"

docker-compose/.env.demo

@@ -34,9 +34,24 @@ DEFAULT_PROXY_DOMAIN=yandex.ru
 # ALLOWED_SNI_LIST=
 ALLOWED_SNI_LIST=vprok.ru,vk.com,perekrestok.ru,x5.ru,yandex.ru,yandex.com,max.ru,alfabank.ru,ozone.ru,rutube.ru
 
+# Block ads and trackers
+# (accepted values: true or false; enabled unless set to false).
+# A TLS handshake whose SNI is a listed domain (or a subdomain of one) is
+# dropped, and so is every packet addressed to an IP such a domain resolves to.
+ENABLE_ADS_FILTER=true
+
+# Comma-separated URLs of the ad/tracker domain lists. Each one is cached in
+# DATA_DIR and downloaded again once the cached copy is older than three days.
+# Empty (default) uses the built-in URLs.
+ADS_BLOCKLIST_URLS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/ultimate-onlydomains.txt,https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
+
+# Directory for the files the server downloads at runtime (the ad/tracker
+# domain lists). It lives on the mounted volume, so the cache survives restarts.
+DATA_DIR=/etc/fptn/data
+
 # Block BitTorrent traffic to prevent abuse
 # (accepted values: true or false; enabled unless set to false)
-DISABLE_TORRENT_FILTER=false
+ENABLE_TORRENT_FILTER=false
 
 # Block the client traffic that gets this server blacklisted
 # (accepted values: true or false; enabled unless set to false). It drops:
@@ -47,11 +62,18 @@ DISABLE_TORRENT_FILTER=false
 #   - amplification reflectors: UDP 1900, 11211
 # NOTE: the mail part also stops desktop mail clients (Thunderbird, Outlook)
 # of your legitimate users from sending mail through the tunnel.
-DISABLE_SPAM_FILTER=false
+ENABLE_SPAM_FILTER=false
+
+# Block the blacklisted domains: a TLS handshake whose SNI is a listed domain
+# (or a subdomain of one) is dropped, and so are the QUIC and ICMP packets
+# addressed to an IP such a domain resolves to
+# (accepted values: true or false; enabled unless set to false).
+ENABLE_DOMAIN_BLACKLIST_FILTER=true
 
-# URL of the domain blacklist, downloaded on every container start.
-# Set it to empty to use only the built-in list.
-BLACKLIST_URL=https://raw.githubusercontent.com/fptn-project/fptn/refs/heads/master/deploy/domain_blacklist/russia.txt
+# Comma-separated URLs of the domain lists to block. Each one is cached in
+# DATA_DIR/blacklist and downloaded again once the cached copy is older than
+# an hour. Empty uses the built-in URLs.
+DOMAIN_BLACKLIST_URLS=https://raw.githubusercontent.com/fptn-project/fptn/refs/heads/master/deploy/domain_blacklist/russia.txt
 
 # Maximum IP packet size (actual packet size will be padded to 1450 due to obfuscation)
 MTU_SIZE=1400

docker-compose/README.md

@@ -80,9 +80,13 @@ services:
       - ENABLE_DETECT_PROBING=${ENABLE_DETECT_PROBING}
       - DEFAULT_PROXY_DOMAIN=${DEFAULT_PROXY_DOMAIN}
       - ALLOWED_SNI_LIST=${ALLOWED_SNI_LIST}
-      - DISABLE_TORRENT_FILTER=${DISABLE_TORRENT_FILTER}
-      - DISABLE_SPAM_FILTER=${DISABLE_SPAM_FILTER}
-      - BLACKLIST_URL=${BLACKLIST_URL:-}
+      - ENABLE_DOMAIN_BLACKLIST_FILTER=${ENABLE_DOMAIN_BLACKLIST_FILTER:-true}
+      - DOMAIN_BLACKLIST_URLS=${DOMAIN_BLACKLIST_URLS:-}
+      - ENABLE_ADS_FILTER=${ENABLE_ADS_FILTER:-true}
+      - ADS_BLOCKLIST_URLS=${ADS_BLOCKLIST_URLS:-}
+      - DATA_DIR=${DATA_DIR:-/etc/fptn/data}
+      - ENABLE_TORRENT_FILTER=${ENABLE_TORRENT_FILTER}
+      - ENABLE_SPAM_FILTER=${ENABLE_SPAM_FILTER}
       - PROMETHEUS_SECRET_ACCESS_KEY=${PROMETHEUS_SECRET_ACCESS_KEY}
       - USE_REMOTE_SERVER_AUTH=${USE_REMOTE_SERVER_AUTH}
       - REMOTE_SERVER_AUTH_HOST=${REMOTE_SERVER_AUTH_HOST}
@@ -156,9 +160,24 @@ DEFAULT_PROXY_DOMAIN=rutube.ru
 # ALLOWED_SNI_LIST=vprok.ru,vk.com,perekrestok.ru,x5.ru,yandex.ru,yandex.com,max.ru,alfabank.ru,ozone.ru,rutube.ru
 ALLOWED_SNI_LIST=
 
+# Block ads and trackers
+# (accepted values: true or false; enabled unless set to false).
+# A TLS handshake whose SNI is a listed domain (or a subdomain of one) is
+# dropped, and so is every packet addressed to an IP such a domain resolves to.
+ENABLE_ADS_FILTER=true
+
+# Comma-separated URLs of the ad/tracker domain lists. Each one is cached in
+# DATA_DIR and downloaded again once the cached copy is older than three days.
+# Empty (default) uses the built-in URLs.
+ADS_BLOCKLIST_URLS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/ultimate-onlydomains.txt,https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
+
+# Directory for the files the server downloads at runtime (the ad/tracker
+# domain lists). It lives on the mounted volume, so the cache survives restarts.
+DATA_DIR=/etc/fptn/data
+
 # Block BitTorrent traffic to prevent abuse
 # (accepted values: true or false; enabled unless set to false)
-DISABLE_TORRENT_FILTER=true
+ENABLE_TORRENT_FILTER=true
 
 # Block the client traffic that gets this server blacklisted
 # (accepted values: true or false; enabled unless set to false). It drops:
@@ -169,12 +188,18 @@ DISABLE_TORRENT_FILTER=true
 #   - amplification reflectors: UDP 1900, 11211
 # NOTE: the mail part also stops desktop mail clients (Thunderbird, Outlook)
 # of your legitimate users from sending mail through the tunnel.
-DISABLE_SPAM_FILTER=true
-
-# URL of the domain blacklist, downloaded on every container start.
-# Traffic to the addresses a listed domain or any of its subdomains resolves
-# to is dropped. Set it to empty to use only the built-in list.
-BLACKLIST_URL=https://raw.githubusercontent.com/fptn-project/fptn/refs/heads/master/deploy/domain_blacklist/russia.txt
+ENABLE_SPAM_FILTER=true
+
+# Block the blacklisted domains: a TLS handshake whose SNI is a listed domain
+# (or a subdomain of one) is dropped, and so are the QUIC and ICMP packets
+# addressed to an IP such a domain resolves to
+# (accepted values: true or false; enabled unless set to false).
+ENABLE_DOMAIN_BLACKLIST_FILTER=true
+
+# Comma-separated URLs of the domain lists to block. Each one is cached in
+# DATA_DIR/blacklist and downloaded again once the cached copy is older than
+# an hour. Empty uses the built-in URLs.
+DOMAIN_BLACKLIST_URLS=https://raw.githubusercontent.com/fptn-project/fptn/refs/heads/master/deploy/domain_blacklist/russia.txt
 
 # Set the USE_REMOTE_SERVER_AUTH variable to true if you need to
 # redirect requests to a master FPTN server for authorization.

docker-compose/docker-compose.yml

@@ -36,9 +36,13 @@ services:
       - ENABLE_DETECT_PROBING=${ENABLE_DETECT_PROBING}
       - DEFAULT_PROXY_DOMAIN=${DEFAULT_PROXY_DOMAIN}
       - ALLOWED_SNI_LIST=${ALLOWED_SNI_LIST}
-      - DISABLE_TORRENT_FILTER=${DISABLE_TORRENT_FILTER}
-      - DISABLE_SPAM_FILTER=${DISABLE_SPAM_FILTER}
-      - BLACKLIST_URL=${BLACKLIST_URL:-}
+      - ENABLE_DOMAIN_BLACKLIST_FILTER=${ENABLE_DOMAIN_BLACKLIST_FILTER:-true}
+      - DOMAIN_BLACKLIST_URLS=${DOMAIN_BLACKLIST_URLS:-}
+      - ENABLE_ADS_FILTER=${ENABLE_ADS_FILTER:-true}
+      - ADS_BLOCKLIST_URLS=${ADS_BLOCKLIST_URLS:-}
+      - DATA_DIR=${DATA_DIR:-/etc/fptn/data}
+      - ENABLE_TORRENT_FILTER=${ENABLE_TORRENT_FILTER}
+      - ENABLE_SPAM_FILTER=${ENABLE_SPAM_FILTER}
       - PROMETHEUS_SECRET_ACCESS_KEY=${PROMETHEUS_SECRET_ACCESS_KEY}
       - USE_REMOTE_SERVER_AUTH=${USE_REMOTE_SERVER_AUTH}
       - REMOTE_SERVER_AUTH_HOST=${REMOTE_SERVER_AUTH_HOST}

src/fptn-server/CMakeLists.txt

@@ -26,11 +26,17 @@ find_package(fmt REQUIRED)
 find_package(nlohmann_json REQUIRED)
 find_package(jwt-cpp REQUIRED)
 find_package(prometheus-cpp REQUIRED)
+find_package(httplib REQUIRED)
 
 
 # Include Boost directory
 include_directories(${Boost_INCLUDE_DIRS})
 
+set(PLATFORM_SPECIFIC_LIBS "")
+if(APPLE)
+  set(PLATFORM_SPECIFIC_LIBS "-framework Security" "-framework CoreFoundation")
+endif()
+
 add_executable(
   "${PROJECT_NAME}"
   fptn-server.cpp
@@ -68,6 +74,10 @@ add_executable(
   filter/filters/bittorrent/bittorrent.cpp
   filter/filters/domain_blacklist/domain_blacklist.h
   filter/filters/domain_blacklist/domain_blacklist.cpp
+  filter/filters/adblock/adblock.h
+  filter/filters/adblock/adblock.cpp
+  filter/domain_list/domain_list.h
+  filter/domain_list/domain_list.cpp
   vpn/manager.h
   vpn/manager.cpp
   statistic/metrics.h
@@ -89,9 +99,11 @@ target_link_libraries(
           OpenSSL::Crypto
           argparse::argparse
           nlohmann_json::nlohmann_json
+          httplib::httplib
           jwt-cpp::jwt-cpp
           spdlog::spdlog
           fmt::fmt
           tuntap++
           fptn-protocol-lib_static
-          prometheus-cpp::prometheus-cpp)
+          prometheus-cpp::prometheus-cpp
+          ${PLATFORM_SPECIFIC_LIBS})

src/fptn-server/config/server_config.cpp

@@ -129,22 +129,72 @@ ServerConfig::ServerConfig(int argc, char* argv[])
         return v.empty() ? "/etc/fptn/users.list" : v;
       });
   // Packet filters
+  args_.add_argument("--enable-domain-blacklist-filter")
+      .help(
+          "Block the blacklisted domains: a TLS handshake whose SNI is a "
+          "listed domain (or a subdomain of one) is dropped, and so are the "
+          "QUIC and ICMP packets addressed to an IP such a domain resolves "
+          "to. Enabled by default, set to 'false' to let this traffic "
+          "through.")
+      .default_value("true")
+      .action([](const std::string& v) -> std::string {
+        return v.empty() ? "true" : v;
+      });
+  args_.add_argument("--domain-blacklist-urls")
+      .help(
+          "Comma-separated URLs of the domain lists to block, one domain per "
+          "line ('#' starts a comment). Every list is cached in --data-dir and "
+          "downloaded again once it is older than an hour. The lists extend "
+          "the built-in one.")
+      .default_value(FPTN_SERVER_DEFAULT_DOMAIN_BLACKLIST_URLS)
+      .action([](const std::string& v) -> std::string {
+        return v.empty() ? FPTN_SERVER_DEFAULT_DOMAIN_BLACKLIST_URLS : v;
+      });
   args_.add_argument("--domain-blacklist-file")
       .help(
           "Path to a file with domains to block, one per line ('#' starts a "
           "comment). Traffic to the addresses a listed domain or any of its "
           "subdomains resolves to is dropped. The list extends the built-in "
           "one. Empty (default) uses only the built-in list.")
       .default_value("");
-  args_.add_argument("--disable-torrent-filter")
+  args_.add_argument("--enable-ads-filter")
+      .help(
+          "Block ads and trackers using the domain lists downloaded from "
+          "--ads-blocklist-urls. A TLS handshake whose SNI is a listed domain "
+          "(or a subdomain of one) is dropped, and every packet addressed to "
+          "an IP such a domain resolves to is dropped as well. Enabled by "
+          "default, set to 'false' to let this traffic through.")
+      .default_value("true")
+      .action([](const std::string& v) -> std::string {
+        return v.empty() ? "true" : v;
+      });
+  args_.add_argument("--ads-blocklist-urls")
+      .help(
+          "Comma-separated URLs of the ad/tracker domain lists (the hosts "
+          "format and a bare domain per line are both accepted). Every list is "
+          "cached in --data-dir and downloaded again once it is older than an "
+          "hour.")
+      .default_value(FPTN_SERVER_DEFAULT_ADS_BLOCKLIST_URLS)
+      .action([](const std::string& v) -> std::string {
+        return v.empty() ? FPTN_SERVER_DEFAULT_ADS_BLOCKLIST_URLS : v;
+      });
+  args_.add_argument("--data-dir")
+      .help(
+          "Directory for the files the server downloads at runtime, such as "
+          "the ad/tracker domain lists (default: /etc/fptn/data)")
+      .default_value("/etc/fptn/data")
+      .action([](const std::string& v) -> std::string {
+        return v.empty() ? "/etc/fptn/data" : v;
+      });
+  args_.add_argument("--enable-torrent-filter")
       .help(
           "Block BitTorrent traffic. Enabled by default, set to 'false' to let "
           "BitTorrent packets through.")
       .default_value("true")
       .action([](const std::string& v) -> std::string {
         return v.empty() ? "true" : v;
       });
-  args_.add_argument("--disable-spam-filter")
+  args_.add_argument("--enable-spam-filter")
       .help(
           "Block the client traffic that gets this server blacklisted: "
           "outgoing mail (the SMTP ports, any TCP stream that starts with an "
@@ -279,12 +329,35 @@ std::string ServerConfig::UserFile() const {
   return args_.get<std::string>("--userfile");
 }
 
-bool ServerConfig::DisableTorrentFilter() const {
-  return ParseBoolean(args_.get<std::string>("--disable-torrent-filter"));
+bool ServerConfig::EnableAdsFilter() const {
+  return ParseBoolean(args_.get<std::string>("--enable-ads-filter"));
+}
+
+std::vector<std::string> ServerConfig::AdsBlocklistUrls() const {
+  return fptn::common::utils::SplitCommaSeparated(
+      args_.get<std::string>("--ads-blocklist-urls"));
+}
+
+std::string ServerConfig::DataDir() const {
+  return args_.get<std::string>("--data-dir");
+}
+
+bool ServerConfig::EnableTorrentFilter() const {
+  return ParseBoolean(args_.get<std::string>("--enable-torrent-filter"));
+}
+
+bool ServerConfig::EnableSpamFilter() const {
+  return ParseBoolean(args_.get<std::string>("--enable-spam-filter"));
+}
+
+bool ServerConfig::EnableDomainBlacklistFilter() const {
+  return ParseBoolean(
+      args_.get<std::string>("--enable-domain-blacklist-filter"));
 }
 
-bool ServerConfig::DisableSpamFilter() const {
-  return ParseBoolean(args_.get<std::string>("--disable-spam-filter"));
+std::vector<std::string> ServerConfig::DomainBlacklistUrls() const {
+  return fptn::common::utils::SplitCommaSeparated(
+      args_.get<std::string>("--domain-blacklist-urls"));
 }
 
 std::string ServerConfig::DomainBlacklistFile() const {

src/fptn-server/config/server_config.h

@@ -42,8 +42,13 @@ class ServerConfig {
   [[nodiscard]] std::uint32_t TunInterfaceNetworkIPv6Mask() const;
 
   [[nodiscard]] std::string UserFile() const;
-  [[nodiscard]] bool DisableTorrentFilter() const;
-  [[nodiscard]] bool DisableSpamFilter() const;
+  [[nodiscard]] bool EnableAdsFilter() const;
+  [[nodiscard]] std::vector<std::string> AdsBlocklistUrls() const;
+  [[nodiscard]] std::string DataDir() const;
+  [[nodiscard]] bool EnableTorrentFilter() const;
+  [[nodiscard]] bool EnableSpamFilter() const;
+  [[nodiscard]] bool EnableDomainBlacklistFilter() const;
+  [[nodiscard]] std::vector<std::string> DomainBlacklistUrls() const;
   [[nodiscard]] std::string DomainBlacklistFile() const;
   [[nodiscard]] std::string PrometheusAccessKey() const;