ZCU-PUB/Add a configurable noindex meta tag for item pages

dataquest-dev/dspace-customers#851 - ZCU was asked to keep two bachelor
theses out of search engine results while they stay fully visible and
searchable inside DSpace and on portal.zcu.cz. That is noindex, not
withdraw and not discoverable=false.

Adds a generic mechanism rather than hardcoding those items:

- item.noIndex config list of item UUIDs / handles. Handle URLs, casing and
  surrounding whitespace are normalized, so 11025/9501,
  http://hdl.handle.net/11025/9501 and the item uuid all match.
- A matching item, or any non-discoverable item, gets
  <meta name="robots" content="noindex, noarchive">.
- No nofollow on purpose: crawlers must stay free to follow the bitstream
  links so they pick up the X-Robots-Tag served for the files themselves.
- citation_pdf_url is dropped for such items, so Google Scholar is never
  handed the full text. All other citation_*/dataset_* tags are kept, so
  Zotero, CRIS harvesters and portal.zcu.cz keep working.

Emitted from setDSOMetaTags() via addMetaTag(), which matters twice: the tag
is server-side rendered, so a crawler sees it in the first HTML response
rather than after hydration; and it is registered in the meta tag store, so
clearMetaTags() removes it on the next route change instead of leaking the
noindex onto every subsequently visited page.

Shipped with an EMPTY list. The merged config is transferred to the browser
unsanitized (server-init.service.ts saveAppConfigForCSR), so anything listed
here becomes publicly readable in the HTML source of every page - it would
publish exactly which items were suppressed. The two #851 items are
de-indexed by the companion X-Robots-Tag nginx map in dspace-customers
instead, which is needed for the PDFs anyway.

Guards against a malformed config value: a YAML scalar (noIndex: 11025/9501)
or a numeric entry would otherwise throw from the first statement of
setDSOMetaTags() and strip the title, description and every citation_* tag
from every item, collection and community page in the repository.

Tests: metadata.service.spec.ts 31 SUCCESS (19 pre-existing + 12 new).
Verified load-bearing: reverting the production change turns 6 of them red.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: milanmajchrak

config/config.example.yml

@@ -286,6 +286,14 @@ item:
     undoTimeout: 10000 # 10 seconds
   # Show the item access status label in items lists
   showAccessStatuses: false
+  # Items that must not be picked up by search engines. Their item page gets
+  # <meta name="robots" content="noindex, noarchive"> and their citation_pdf_url tag is dropped;
+  # the items stay fully visible and searchable inside DSpace (this is not withdraw and not
+  # discoverable=false). Entries are item UUIDs or handles ("123456789/42"); the
+  # http(s)://hdl.handle.net/ prefix, casing and whitespace are ignored.
+  # Non-discoverable (private) items always get the tag, independently of this list.
+  # This array REPLACES the default, it does not append to it.
+  noIndex: []
   bitstream:
     # Number of entries in the bitstream list in the item view page.
     # Rounded to the nearest size in the list of selectable sizes on the

config/config.yml

@@ -192,3 +192,22 @@ item:
   bitstream:
     # Per-bitstream embargo-date badge in the Item View
     showAccessStatuses: true
+  # Items that must NOT be picked up by search engines - dataquest-dev/dspace-customers#851.
+  # Such an item stays fully visible and searchable inside DSpace and on portal.zcu.cz: this is
+  # noindex only, NOT withdraw and NOT discoverable=false. It gets
+  # <meta name="robots" content="noindex, noarchive"> (server-side rendered, so crawlers see it in
+  # the first HTML response) and its citation_pdf_url tag is dropped.
+  #
+  # Entries may be item UUIDs or handles; the http(s)://hdl.handle.net/ prefix, casing and
+  # surrounding whitespace are ignored. Empty list = feature off.
+  #
+  # DELIBERATELY EMPTY. This config is transferred to the browser unsanitized
+  # (src/modules/app/server-init.service.ts saveAppConfigForCSR), so anything listed here becomes
+  # publicly readable in the HTML source of every page - i.e. it would publish exactly which items
+  # were suppressed. On a takedown-flavoured request that is not acceptable, so the #851 items are
+  # de-indexed by the X-Robots-Tag nginx map instead (dspace-customers, customer-specific/zcu/bits).
+  # Use this list only for cases where publishing the identifiers is fine.
+  #
+  # NOTE: config merging REPLACES this array (deepmerge: the source array wins), it does not
+  # append - always list every item that must stay noindexed.
+  noIndex: []

src/app/core/metadata/metadata.service.spec.ts

@@ -12,8 +12,10 @@ import {
   ItemMock,
   MockBitstream1,
   MockBitstream3,
-  MockBitstream2
+  MockBitstream2,
+  NonDiscoverableItemMock
 } from '../../shared/mocks/item.mock';
+import { DSpaceObject } from '../shared/dspace-object.model';
 import { createSuccessfulRemoteDataObject, createSuccessfulRemoteDataObject$ } from '../../shared/remote-data.utils';
 import { PaginatedList } from '../data/paginated-list.model';
 import { Bitstream } from '../shared/bitstream.model';
@@ -96,6 +98,7 @@ describe('MetadataService', () => {
 
     appConfig = {
       item: {
+        noIndex: [],
         bitstream: {
           pageSize: 5
         }
@@ -406,6 +409,93 @@ describe('MetadataService', () => {
     });
   });
 
+  describe('robots meta tag', () => {
+    const noIndexTag = { name: 'robots', content: 'noindex, noarchive' };
+
+    const routeTo = (dso: any) => {
+      (metadataService as any).processRouteChange({
+        data: { value: { dso: createSuccessfulRemoteDataObject(dso) } }
+      });
+      tick();
+    };
+
+    it('should not add a robots tag for a normal discoverable item', fakeAsync(() => {
+      routeTo(ItemMock);
+      expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' }));
+    }));
+
+    it('should add a robots noindex tag for a non-discoverable item', fakeAsync(() => {
+      routeTo(NonDiscoverableItemMock);
+      expect(meta.addTag).toHaveBeenCalledWith(noIndexTag);
+    }));
+
+    it('should add a robots noindex tag when the item uuid is configured', fakeAsync(() => {
+      appConfig.item.noIndex = ['0ec7ff22-f211-40ab-a69e-c819b0b1f357'];
+      routeTo(ItemMock);
+      expect(meta.addTag).toHaveBeenCalledWith(noIndexTag);
+    }));
+
+    it('should add a robots noindex tag when the item handle is configured', fakeAsync(() => {
+      appConfig.item.noIndex = ['10673/6'];
+      routeTo(ItemMock);
+      expect(meta.addTag).toHaveBeenCalledWith(noIndexTag);
+    }));
+
+    it('should normalize handle URLs, casing and whitespace in item.noIndex', fakeAsync(() => {
+      appConfig.item.noIndex = ['  HTTP://hdl.handle.net/10673/6  '];
+      routeTo(ItemMock);
+      expect(meta.addTag).toHaveBeenCalledWith(noIndexTag);
+    }));
+
+    it('should not add a robots tag for an item that is not configured', fakeAsync(() => {
+      appConfig.item.noIndex = ['11025/9501', 'f4c45569-cdfc-4b3d-98df-46bfeba016b9'];
+      routeTo(ItemMock);
+      expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' }));
+    }));
+
+    it('should not add a robots tag for a non-Item DSpaceObject', fakeAsync(() => {
+      appConfig.item.noIndex = ['10673/6'];
+      routeTo(Object.assign(new DSpaceObject(), { uuid: '10673/6', handle: '10673/6', metadata: {} }));
+      expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' }));
+    }));
+
+    it('should register the robots tag in the meta tag store so it is cleared on the next route change', fakeAsync(() => {
+      appConfig.item.noIndex = ['10673/6'];
+      routeTo(ItemMock);
+      expect(store.dispatch).toHaveBeenCalledWith(new AddMetaTagAction('robots'));
+    }));
+
+    it('should suppress citation_pdf_url for a noindex item', fakeAsync(() => {
+      appConfig.item.noIndex = ['10673/6'];
+      routeTo(ItemMock);
+      expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_pdf_url' }));
+      expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_title' }));
+    }));
+
+    it('should keep citation_pdf_url for a normal item', fakeAsync(() => {
+      routeTo(ItemMock);
+      expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_pdf_url' }));
+    }));
+
+    it('should not break the other meta tags when item.noIndex is a scalar instead of a list', fakeAsync(() => {
+      // A YAML scalar (noIndex: 11025/9501) is the natural single-value form and an easy slip,
+      // since every other key under item: is a scalar. It must degrade to "feature off" and never
+      // throw: setNoIndexTag() is the first statement of setDSOMetaTags(), so a throw here would
+      // strip the title, description and all citation_* tags from every page in the repository.
+      appConfig.item.noIndex = '10673/6' as any;
+      expect(() => routeTo(ItemMock)).not.toThrow();
+      expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_title' }));
+      expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' }));
+    }));
+
+    it('should ignore non-string entries in item.noIndex without throwing', fakeAsync(() => {
+      appConfig.item.noIndex = [9501 as any, null, '10673/6'];
+      expect(() => routeTo(ItemMock)).not.toThrow();
+      expect(meta.addTag).toHaveBeenCalledWith(noIndexTag);
+      expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_title' }));
+    }));
+  });
+
   describe(`when there's no bitstream with an allowed format on the first page`, () => {
     let bitstreams;
 

src/app/core/metadata/metadata.service.ts

@@ -63,6 +63,14 @@ const tagsInUseSelector =
     (state: MetaTagState) => state.tagsInUse,
   );
 
+/**
+ * Content of the robots meta tag emitted for items that must not be indexed by search engines.
+ * `noindex` removes the page from search results, `noarchive` also suppresses cached copies.
+ * Deliberately NO `nofollow`: crawlers must stay free to follow the bitstream links so they can
+ * pick up the `X-Robots-Tag: noindex` served for the files themselves.
+ */
+export const NO_INDEX_META_CONTENT = 'noindex, noarchive';
+
 @Injectable()
 export class MetadataService {
 
@@ -147,6 +155,8 @@ export class MetadataService {
 
   private setDSOMetaTags(): void {
 
+    this.setNoIndexTag();
+
     this.setTitleTag();
     this.setDescriptionTag();
 
@@ -194,6 +204,61 @@ export class MetadataService {
 
   }
 
+  /**
+   * Add <meta name="robots" content="noindex, noarchive"> to the <head> for Items that must not
+   * be picked up by search engines.
+   *
+   * Added through addMetaTag() on purpose: that registers the tag name in the meta tag store, so
+   * clearMetaTags() removes it again at the top of the next processRouteChange(). Adding it via
+   * this.meta.addTag() directly (as setGenerator() does) would leak the noindex onto every
+   * subsequently visited page.
+   */
+  protected setNoIndexTag(): void {
+    if (this.isNoIndex()) {
+      this.addMetaTag('robots', NO_INDEX_META_CONTENT);
+    }
+  }
+
+  /**
+   * Whether the currently resolved DSpaceObject must be excluded from search engine indexes:
+   * either it is a non-discoverable (private) Item, or its uuid/handle is listed in the
+   * `item.noIndex` configuration. See dataquest-dev/dspace-customers#851.
+   *
+   * Single predicate on purpose, so the trigger can later be replaced by (or extended with) an
+   * item metadata field such as local.noindex without touching any call site.
+   */
+  private isNoIndex(): boolean {
+    if (!(this.currentObject.value instanceof Item)) {
+      return false;
+    }
+    const item = this.currentObject.value as Item;
+    if (item.isDiscoverable === false) {
+      return true;
+    }
+    const configured = this.appConfig?.item?.noIndex;
+    // Array.isArray, not hasNoValue: a YAML scalar (noIndex: 11025/9501) also has a length, and
+    // throwing here would strip the title, description and every citation_* tag from every item,
+    // collection and community page in the repository - see the first statement of setDSOMetaTags().
+    if (!Array.isArray(configured) || configured.length === 0) {
+      return false;
+    }
+    const itemIds = [item.uuid, item.handle]
+      .filter((id) => isNotEmpty(id))
+      .map((id) => this.normalizeNoIndexId(id));
+    return configured.some((id: any) => typeof id === 'string' && isNotEmpty(id)
+      && itemIds.includes(this.normalizeNoIndexId(id)));
+  }
+
+  /**
+   * Normalize an identifier so operators can configure a bare handle (11025/9501), a handle URL
+   * (http://hdl.handle.net/11025/9501) or an item uuid, in any casing and with stray whitespace.
+   */
+  private normalizeNoIndexId(id: string): string {
+    return id.trim().toLowerCase()
+      .replace(/^https?:\/\/hdl\.handle\.net\//, '')
+      .replace(/^\/+/, '');
+  }
+
   /**
    * Add <meta name="title" ... >  to the <head>
    */
@@ -349,6 +414,13 @@ export class MetadataService {
    * Add <meta name="citation_pdf_url" ... >  to the <head>
    */
   private setCitationPdfUrlTag(): void {
+    if (this.isNoIndex()) {
+      // Never hand the full text to Google Scholar for items that must not be indexed: Scholar is
+      // a separate crawl pipeline that keys off citation_pdf_url and does not reliably honour a
+      // landing page <meta name="robots">. All other citation_*/dataset_* tags are kept, so
+      // Zotero, CRIS harvesters and portal.zcu.cz keep working.
+      return;
+    }
     if (this.currentObject.value instanceof Item) {
       const item = this.currentObject.value as Item;
 

src/app/shared/mocks/item.mock.ts

@@ -294,3 +294,17 @@ export const ItemMock: Item = Object.assign(new Item(), {
   )
 });
 /* eslint-enable @typescript-eslint/no-shadow */
+
+/**
+ * An Item that is not discoverable (private), mirroring upstream dspace-7_x.
+ * `metadata` is copied into a fresh object on purpose: spec helpers such as mockType()/
+ * mockPublisher() mutate `metadata` in place, so a shared reference would cross-contaminate
+ * this mock and {@link ItemMock}.
+ */
+export const NonDiscoverableItemMock: Item = Object.assign(new Item(), ItemMock, {
+  handle: '10673/7',
+  id: '0ec7ff22-f211-40ab-a69e-c819b0b1f358',
+  uuid: '0ec7ff22-f211-40ab-a69e-c819b0b1f358',
+  isDiscoverable: false,
+  metadata: Object.assign({}, ItemMock.metadata),
+});

src/config/default-app-config.ts

@@ -281,6 +281,9 @@ export class DefaultAppConfig implements AppConfig {
     },
     // Show the item access status label in items lists
     showAccessStatuses: false,
+    // Items excluded from search engine indexes (uuids or handles). Empty by default so this is
+    // a no-op for every instance that does not configure it; see item-config.interface.ts.
+    noIndex: [],
     bitstream: {
       // Number of entries in the bitstream list in the item view page.
       // Rounded to the nearest size in the list of selectable sizes on the

src/config/item-config.interface.ts

@@ -7,6 +7,13 @@ export interface ItemConfig extends Config {
   // This is used to show the access status label of items in results lists
   showAccessStatuses: boolean;
 
+  // Items that must not be picked up by search engines: their item page gets
+  // <meta name="robots" content="noindex, noarchive"> and their citation_pdf_url tag is dropped.
+  // Entries are item UUIDs or handles ("11025/9501"); a http(s)://hdl.handle.net/ prefix, casing
+  // and surrounding whitespace are ignored. Empty list = feature off.
+  // Non-discoverable (private) items always get the tag, independently of this list.
+  noIndex: string[];
+
   bitstream: {
     // Number of entries in the bitstream list in the item view page.
     // Rounded to the nearest size in the list of selectable sizes on the

src/environments/environment.test.ts

@@ -250,6 +250,8 @@ export const environment: BuildConfig = {
     },
     // Show the item access status label in items lists
     showAccessStatuses: false,
+    // Items excluded from search engine indexes (uuids or handles)
+    noIndex: [],
     bitstream: {
       // Number of entries in the bitstream list in the item view page.
       // Rounded to the nearest size in the list of selectable sizes on the