fix(ssr): drop dead scoped-search patterns, add robots.txt test

Copilot review (and an independent verification pass) found that the two
scoped-search SSR excludes and their matching robots.txt lines never match
anything on this 7.6.5 branch: collection/community routing has no `:id/search`
child route here (that shape is DSpace 9.x, where the mendelu source commit
came from), and scoped search on 7.6.5 is `/search?scope=<uuid>&f.*` — already
covered by the pre-existing `Disallow: /search` and `^/search` SSR exclusion.

- Remove the dead `^/(communities|collections)/[uuid]/search` patterns from
  config.yml, config.example.yml, environment.ts, environment.production.ts,
  environment.test.ts, and the matching dead robots.txt Disallow lines.
- Keep the effective, new parts: the generic facet trap (Disallow: /*?f. and
  /*&f.) and the three opt-in toggles — those are unaffected by the routing
  question.
- Extract the toggle-line assembly into buildOptionalRobotsDisallows()
  (src/config/robots.util.ts), rendered as a single EJS local from server.ts,
  and add a unit test (robots.util.spec.ts) covering each toggle and the
  no-blank-line-in-group invariant Copilot flagged as untested.

Not applied: rewriting isExcludedFromSsr to inspect facet query keys. The
documented incident (scoped-search facet enumeration) is already covered by
the existing /search SSR exclusion; changing core SSR routing logic for every
request is out of scope for this fix.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: jr-rk

config/config.example.yml

@@ -31,14 +31,6 @@ universal:
       flag: "i"
     - pattern: "^/collections/[a-f0-9-]{36}/browse(/.*)?$"
       flag: "i"
-    # Scoped search inside a community/collection is the most expensive route, and its facet
-    # links are effectively unbounded; crawlers walking them saturated SSR in production
-    # (HTTP 504). Serve these as a plain CSR shell. src/robots.txt.ejs asks well-behaved
-    # crawlers to stay out of the same paths; this handles the ones that ignore robots.txt.
-    - pattern: "^/communities/[a-f0-9-]{36}/search(/.*)?$"
-      flag: "i"
-    - pattern: "^/collections/[a-f0-9-]{36}/search(/.*)?$"
-      flag: "i"
     - pattern: "^/browse/"
     - pattern: "^/search$"
     - pattern: "^/community-list$"

config/config.yml

@@ -19,14 +19,6 @@ universal:
       flag: "i"
     - pattern: "^/collections/[a-f0-9-]{36}/browse(/.*)?$"
       flag: "i"
-    # Scoped search inside a community/collection is the most expensive route, and its facet
-    # links are effectively unbounded; crawlers walking them saturated SSR in production
-    # (HTTP 504). Serve these as a plain CSR shell. src/robots.txt.ejs asks well-behaved
-    # crawlers to stay out of the same paths; this handles the ones that ignore robots.txt.
-    - pattern: "^/communities/[a-f0-9-]{36}/search(/.*)?$"
-      flag: "i"
-    - pattern: "^/collections/[a-f0-9-]{36}/search(/.*)?$"
-      flag: "i"
     - pattern: "^/browse/"
     - pattern: "^/search$"
     - pattern: "^/community-list$"

server.ts

@@ -56,6 +56,7 @@ import { extendEnvironmentWithAppConfig } from './src/config/config.util';
 import { logStartupMessage } from './startup-message';
 import { TOKENITEM } from './src/app/core/auth/models/auth-token-info.model';
 import { SsrExcludePatterns } from './src/config/universal-config.interface';
+import { buildOptionalRobotsDisallows } from './src/config/robots.util';
 
 
 /*
@@ -163,15 +164,13 @@ export function app() {
 
   /**
    * Serve the robots.txt ejs template, filling in the origin variable and the
-   * per-instance optional-block toggles (see config `robots`).
+   * per-instance optional Disallow blocks (see config `robots`).
    */
   server.get('/robots.txt', (req, res) => {
     res.setHeader('content-type', 'text/plain');
     res.render('assets/robots.txt.ejs', {
       'origin': req.protocol + '://' + req.headers.host,
-      'disallowHandle': environment.robots.disallowHandle,
-      'disallowBrowse': environment.robots.disallowBrowse,
-      'disallowBitstreams': environment.robots.disallowBitstreams
+      'optionalDisallows': buildOptionalRobotsDisallows(environment.robots)
     });
   });
 

src/config/robots.util.spec.ts

@@ -0,0 +1,40 @@
+import { buildOptionalRobotsDisallows } from './robots.util';
+import { RobotsConfig } from './robots-config.interface';
+
+describe('buildOptionalRobotsDisallows', () => {
+  const allOff: RobotsConfig = {
+    disallowHandle: false,
+    disallowBrowse: false,
+    disallowBitstreams: false,
+  };
+
+  it('emits nothing when every toggle is off', () => {
+    expect(buildOptionalRobotsDisallows(allOff)).toBe('');
+  });
+
+  it('emits only /browse when disallowBrowse is on', () => {
+    expect(buildOptionalRobotsDisallows({ ...allOff, disallowBrowse: true }))
+      .toBe('\nDisallow: /browse');
+  });
+
+  it('emits only /handle when disallowHandle is on', () => {
+    expect(buildOptionalRobotsDisallows({ ...allOff, disallowHandle: true }))
+      .toBe('\nDisallow: /handle');
+  });
+
+  it('emits both bitstream rules when disallowBitstreams is on', () => {
+    const out = buildOptionalRobotsDisallows({ ...allOff, disallowBitstreams: true });
+    expect(out).toContain('Disallow: /bitstream/');
+    expect(out).toContain('Disallow: /bitstreams/');
+  });
+
+  it('leads with a newline and never a blank line, so the block stays inside the group', () => {
+    const out = buildOptionalRobotsDisallows({
+      disallowHandle: true,
+      disallowBrowse: true,
+      disallowBitstreams: true,
+    });
+    expect(out.startsWith('\n')).toBeTrue();
+    expect(out).not.toContain('\n\n');
+  });
+});

src/config/robots.util.ts

@@ -0,0 +1,27 @@
+import { RobotsConfig } from './robots-config.interface';
+
+/**
+ * Build the optional, per-instance `Disallow` lines for the served robots.txt
+ * (see `src/robots.txt.ejs`) from config. The always-on facet-trap rules live in
+ * the template itself; only these switchable blocks are assembled here so the
+ * toggle logic has a unit-testable seam.
+ *
+ * Returns a string that already starts with a newline when non-empty, so it can
+ * be appended directly after the last static rule without leaving a blank line
+ * inside the `User-agent: *` group record (blank lines end a group).
+ */
+export function buildOptionalRobotsDisallows(robots: RobotsConfig): string {
+  const lines: string[] = [];
+  if (robots?.disallowBrowse) {
+    lines.push('Disallow: /browse');
+  }
+  if (robots?.disallowHandle) {
+    lines.push('Disallow: /handle');
+  }
+  if (robots?.disallowBitstreams) {
+    // Bitstream content is reachable by handle path and by UUID; block both.
+    lines.push('Disallow: /bitstream/');
+    lines.push('Disallow: /bitstreams/');
+  }
+  return lines.length ? '\n' + lines.join('\n') : '';
+}

src/environments/environment.production.ts

@@ -20,15 +20,6 @@ export const environment: Partial<BuildConfig> = {
         pattern: '^/collections/[a-f0-9-]{36}/browse(/.*)?$',
         flag: 'i',
       },
-      // Scoped search + its facet links: the crawler trap that saturated SSR in production.
-      {
-        pattern: '^/communities/[a-f0-9-]{36}/search(/.*)?$',
-        flag: 'i',
-      },
-      {
-        pattern: '^/collections/[a-f0-9-]{36}/search(/.*)?$',
-        flag: 'i',
-      },
       { pattern: '^/browse/' },
       { pattern: '^/search' },
       { pattern: '^/community-list$' },

src/environments/environment.test.ts

@@ -23,15 +23,6 @@ export const environment: BuildConfig = {
         pattern: '^/collections/[a-f0-9-]{36}/browse(/.*)?$',
         flag: 'i',
       },
-      // Scoped search + its facet links: the crawler trap that saturated SSR in production.
-      {
-        pattern: '^/communities/[a-f0-9-]{36}/search(/.*)?$',
-        flag: 'i',
-      },
-      {
-        pattern: '^/collections/[a-f0-9-]{36}/search(/.*)?$',
-        flag: 'i',
-      },
       { pattern: '^/browse/' },
       { pattern: '^/search' },
       { pattern: '^/community-list$' },

src/environments/environment.ts

@@ -25,15 +25,6 @@ export const environment: Partial<BuildConfig> = {
         pattern: '^/collections/[a-f0-9-]{36}/browse(/.*)?$',
         flag: 'i',
       },
-      // Scoped search + its facet links: the crawler trap that saturated SSR in production.
-      {
-        pattern: '^/communities/[a-f0-9-]{36}/search(/.*)?$',
-        flag: 'i',
-      },
-      {
-        pattern: '^/collections/[a-f0-9-]{36}/search(/.*)?$',
-        flag: 'i',
-      },
       { pattern: '^/browse/' },
       { pattern: '^/search' },
       { pattern: '^/community-list$' },

src/robots.txt.ejs

@@ -18,27 +18,12 @@ Disallow: /profile
 Disallow: /workflowitems
 # Crawlers should be able to access entity pages, but not the facet search links present on entity pages
 Disallow: /entities/*?f
-# "Disallow: /search" above only matches paths that START with /search, so scoped
-# search inside a community or collection stayed crawlable. Enumerating its facet
-# links saturated SSR in production (HTTP 504); block it and every facet URL.
-Disallow: /collections/*/search
-Disallow: /communities/*/search
-# Any URL carrying a Discovery facet filter (f.author, f.subject, ...). Two rules
-# because the facet can be the first query parameter (?f.) or a later one (&f.),
-# and robots.txt cannot express "either".
+# Discovery facet links (f.author, f.subject, ...) multiply into an effectively
+# unbounded set of crawlable URLs; enumerating them saturated SSR in production
+# (HTTP 504). Two rules because the facet can be the first query parameter (?f.)
+# or a later one (&f.), and robots.txt cannot express "either".
 Disallow: /*?f.
-Disallow: /*&f.
-<% if (disallowBrowse) { -%>
-Disallow: /browse
-<% } -%>
-<% if (disallowHandle) { -%>
-Disallow: /handle
-<% } -%>
-<% if (disallowBitstreams) { -%>
-# Bitstream content is reachable by handle path and by UUID; block both.
-Disallow: /bitstream/
-Disallow: /bitstreams/
-<% } -%>
+Disallow: /*&f.<%- optionalDisallows %>
 
 # Optionally uncomment the following line ONLY if sitemaps are working
 # and you have verified that your site is being indexed correctly.