[Port to dtq-dev] Issue dspace-customers#902: Add configurable crawler policy (robots.txt + SSR excludes)

config/config.example.yml

@@ -31,6 +31,14 @@ universal:
       flag: "i"
     - pattern: "^/collections/[a-f0-9-]{36}/browse(/.*)?$"
       flag: "i"
+    # Scoped search inside a community/collection is the most expensive route, and its facet
+    # links are effectively unbounded; crawlers walking them saturated SSR in production
+    # (HTTP 504). Serve these as a plain CSR shell. src/robots.txt.ejs asks well-behaved
+    # crawlers to stay out of the same paths; this handles the ones that ignore robots.txt.
+    - pattern: "^/communities/[a-f0-9-]{36}/search(/.*)?$"
+      flag: "i"
+    - pattern: "^/collections/[a-f0-9-]{36}/search(/.*)?$"
+      flag: "i"
     - pattern: "^/browse/"
     - pattern: "^/search$"
     - pattern: "^/community-list$"
@@ -60,6 +68,18 @@ universal:
   # Disable this setting to avoid URL replacement during SSR. In this the state is not transferred to avoid security issues.
   replaceRestUrl: true
 
+# Optional per-instance blocks of the served /robots.txt (src/robots.txt.ejs).
+# The Discovery facet-trap rules are always emitted; these switch the extra blocks.
+# All default false because each de-indexes content some instances intentionally
+# expose (a blanket /handle block breaks instances relying on handle redirects).
+robots:
+  # Disallow: /handle
+  disallowHandle: false
+  # Disallow: /browse
+  disallowBrowse: false
+  # Disallow: /bitstream/ and /bitstreams/
+  disallowBitstreams: false
+
 # The REST API server settings
 # NOTE: these settings define which (publicly available) REST API to use. They are usually
 # 'synced' with the 'dspace.server.url' setting in your backend's local.cfg.

config/config.yml

@@ -19,6 +19,14 @@ universal:
       flag: "i"
     - pattern: "^/collections/[a-f0-9-]{36}/browse(/.*)?$"
       flag: "i"
+    # Scoped search inside a community/collection is the most expensive route, and its facet
+    # links are effectively unbounded; crawlers walking them saturated SSR in production
+    # (HTTP 504). Serve these as a plain CSR shell. src/robots.txt.ejs asks well-behaved
+    # crawlers to stay out of the same paths; this handles the ones that ignore robots.txt.
+    - pattern: "^/communities/[a-f0-9-]{36}/search(/.*)?$"
+      flag: "i"
+    - pattern: "^/collections/[a-f0-9-]{36}/search(/.*)?$"
+      flag: "i"
     - pattern: "^/browse/"
     - pattern: "^/search$"
     - pattern: "^/community-list$"
@@ -38,6 +46,15 @@ universal:
   # If set to false the component will not be included in the HTML returned from the server side rendering.
   enableBrowseComponent: false
 
+# Optional per-instance blocks of the served /robots.txt (src/robots.txt.ejs).
+# The Discovery facet-trap rules are always emitted; these switch the extra blocks.
+# All default false because each de-indexes content some instances intentionally
+# expose (a blanket /handle block breaks instances relying on handle redirects).
+robots:
+  disallowHandle: false
+  disallowBrowse: false
+  disallowBitstreams: false
+
 # Caching settings
 cache:
   # NOTE: how long should objects be cached for by default

server.ts

@@ -162,12 +162,16 @@ export function app() {
   server.set('view engine', 'ejs');
 
   /**
-   * Serve the robots.txt ejs template, filling in the origin variable
+   * Serve the robots.txt ejs template, filling in the origin variable and the
+   * per-instance optional-block toggles (see config `robots`).
    */
   server.get('/robots.txt', (req, res) => {
     res.setHeader('content-type', 'text/plain');
     res.render('assets/robots.txt.ejs', {
-      'origin': req.protocol + '://' + req.headers.host
+      'origin': req.protocol + '://' + req.headers.host,
+      'disallowHandle': environment.robots.disallowHandle,
+      'disallowBrowse': environment.robots.disallowBrowse,
+      'disallowBitstreams': environment.robots.disallowBitstreams
     });
   });
 

src/config/app-config.interface.ts

@@ -27,6 +27,7 @@ import { SearchConfig } from './search-page-config.interface';
 import { AccessibilitySettingsConfig } from '../app/accessibility/accessibility-settings.config';
 import { MatomoConfig } from './matomo-config';
 import { StatisticsConfig } from './statistics-config';
+import { RobotsConfig } from './robots-config.interface';
 
 interface AppConfig extends Config {
   ui: UIServerConfig;
@@ -59,6 +60,7 @@ interface AppConfig extends Config {
   signpostingEnabled: boolean;
   matomo: MatomoConfig;
   statistics?: StatisticsConfig;
+  robots: RobotsConfig;
 }
 
 /**

src/config/default-app-config.ts

@@ -26,6 +26,7 @@ import { LiveRegionConfig } from '../app/shared/live-region/live-region.config';
 import { SearchConfig } from './search-page-config.interface';
 import { AccessibilitySettingsConfig } from '../app/accessibility/accessibility-settings.config';
 import { MatomoConfig } from './matomo-config';
+import { RobotsConfig } from './robots-config.interface';
 
 export class DefaultAppConfig implements AppConfig {
   production = false;
@@ -415,6 +416,15 @@ export class DefaultAppConfig implements AppConfig {
     mathjax: false,
   };
 
+  // Optional per-instance blocks of the served robots.txt. The facet-trap rules
+  // that protect SSR are always emitted; these switch the extra blocks that
+  // de-index content some instances intentionally expose.
+  robots: RobotsConfig = {
+    disallowHandle: false,
+    disallowBrowse: false,
+    disallowBitstreams: false,
+  };
+
   // Which vocabularies should be used for which search filters
   // and whether to show the filter in the search sidebar
   // Take a look at the filter-vocabulary-config.ts file for documentation on how the options are obtained

src/config/robots-config.interface.ts

@@ -0,0 +1,27 @@
+import { Config } from './config.interface';
+
+/**
+ * Config for the optional, per-instance blocks of the served `robots.txt`
+ * (see `src/robots.txt.ejs`). The facet-trap rules that protect SSR are always
+ * emitted; only these extra blocks are switchable, because each de-indexes
+ * content some instances intentionally expose.
+ */
+export interface RobotsConfig extends Config {
+
+  /**
+   * Emit `Disallow: /handle`. Off by default: a blanket handle block de-indexes
+   * persistent identifiers on instances that rely on handle redirects.
+   */
+  disallowHandle: boolean;
+
+  /**
+   * Emit `Disallow: /browse`.
+   */
+  disallowBrowse: boolean;
+
+  /**
+   * Emit `Disallow: /bitstream/` and `Disallow: /bitstreams/` to keep crawlers
+   * out of bitstream content (reachable by handle path or by UUID).
+   */
+  disallowBitstreams: boolean;
+}

src/environments/environment.production.ts

@@ -20,6 +20,15 @@ export const environment: Partial<BuildConfig> = {
         pattern: '^/collections/[a-f0-9-]{36}/browse(/.*)?$',
         flag: 'i',
       },
+      // Scoped search + its facet links: the crawler trap that saturated SSR in production.
+      {
+        pattern: '^/communities/[a-f0-9-]{36}/search(/.*)?$',
+        flag: 'i',
+      },
+      {
+        pattern: '^/collections/[a-f0-9-]{36}/search(/.*)?$',
+        flag: 'i',
+      },
       { pattern: '^/browse/' },
       { pattern: '^/search' },
       { pattern: '^/community-list$' },

src/environments/environment.test.ts

@@ -23,6 +23,15 @@ export const environment: BuildConfig = {
         pattern: '^/collections/[a-f0-9-]{36}/browse(/.*)?$',
         flag: 'i',
       },
+      // Scoped search + its facet links: the crawler trap that saturated SSR in production.
+      {
+        pattern: '^/communities/[a-f0-9-]{36}/search(/.*)?$',
+        flag: 'i',
+      },
+      {
+        pattern: '^/collections/[a-f0-9-]{36}/search(/.*)?$',
+        flag: 'i',
+      },
       { pattern: '^/browse/' },
       { pattern: '^/search' },
       { pattern: '^/community-list$' },
@@ -327,6 +336,11 @@ export const environment: BuildConfig = {
     enabled: false,
     mathjax: false,
   },
+  robots: {
+    disallowHandle: false,
+    disallowBrowse: false,
+    disallowBitstreams: false,
+  },
   comcolSelectionSort: {
     sortField:'dc.title',
     sortDirection:'ASC',

src/environments/environment.ts

@@ -25,6 +25,15 @@ export const environment: Partial<BuildConfig> = {
         pattern: '^/collections/[a-f0-9-]{36}/browse(/.*)?$',
         flag: 'i',
       },
+      // Scoped search + its facet links: the crawler trap that saturated SSR in production.
+      {
+        pattern: '^/communities/[a-f0-9-]{36}/search(/.*)?$',
+        flag: 'i',
+      },
+      {
+        pattern: '^/collections/[a-f0-9-]{36}/search(/.*)?$',
+        flag: 'i',
+      },
       { pattern: '^/browse/' },
       { pattern: '^/search' },
       { pattern: '^/community-list$' },

src/robots.txt.ejs

@@ -18,6 +18,27 @@ Disallow: /profile
 Disallow: /workflowitems
 # Crawlers should be able to access entity pages, but not the facet search links present on entity pages
 Disallow: /entities/*?f
+# "Disallow: /search" above only matches paths that START with /search, so scoped
+# search inside a community or collection stayed crawlable. Enumerating its facet
+# links saturated SSR in production (HTTP 504); block it and every facet URL.
+Disallow: /collections/*/search
+Disallow: /communities/*/search
+# Any URL carrying a Discovery facet filter (f.author, f.subject, ...). Two rules
+# because the facet can be the first query parameter (?f.) or a later one (&f.),
+# and robots.txt cannot express "either".
+Disallow: /*?f.
+Disallow: /*&f.
+<% if (disallowBrowse) { -%>
+Disallow: /browse
+<% } -%>
+<% if (disallowHandle) { -%>
+Disallow: /handle
+<% } -%>
+<% if (disallowBitstreams) { -%>
+# Bitstream content is reachable by handle path and by UUID; block both.
+Disallow: /bitstream/
+Disallow: /bitstreams/
+<% } -%>
 
 # Optionally uncomment the following line ONLY if sitemaps are working
 # and you have verified that your site is being indexed correctly.