ZCU-PUB/Add a configurable noindex meta tag for item pages

Adds item.noIndex - a list of item uuids or handles whose item page gets
<meta name="robots" content="noindex, noarchive"> and has its citation_pdf_url
dropped. Non-discoverable items get the tag too.

Server-side rendered, so crawlers see the tag in the first HTML response.
Emitted via addMetaTag() so clearMetaTags() removes it on the next route
change instead of leaking it onto every page visited afterwards. No nofollow:
crawlers should keep following bitstream links to pick up the X-Robots-Tag
served for the files.

Ships with an empty list on purpose - this config is transferred to the
browser unsanitized, so entries would be publicly readable in every page's
HTML. Guards on Array.isArray() because a misconfigured scalar would otherwise
throw from the first statement of setDSOMetaTags() and strip the meta tags off
every page.

Tests: metadata.service.spec.ts 31 SUCCESS (19 existing + 12 new); reverting
the production change turns 6 of them red.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: milanmajchrak

config/config.example.yml

@@ -286,6 +286,8 @@ item:
     undoTimeout: 10000 # 10 seconds
   # Show the item access status label in items lists
   showAccessStatuses: false
+  # Item uuids or handles to exclude from search engine indexes. Empty = off.
+  noIndex: []
   bitstream:
     # Number of entries in the bitstream list in the item view page.
     # Rounded to the nearest size in the list of selectable sizes on the

config/config.yml

@@ -192,3 +192,6 @@ item:
   bitstream:
     # Per-bitstream embargo-date badge in the Item View
     showAccessStatuses: true
+  # Item uuids or handles to exclude from search engine indexes. Empty = off.
+  # Keep empty: this config is served to the browser, so entries here are public.
+  noIndex: []

src/app/core/metadata/metadata.service.spec.ts

@@ -12,8 +12,10 @@ import {
   ItemMock,
   MockBitstream1,
   MockBitstream3,
-  MockBitstream2
+  MockBitstream2,
+  NonDiscoverableItemMock
 } from '../../shared/mocks/item.mock';
+import { DSpaceObject } from '../shared/dspace-object.model';
 import { createSuccessfulRemoteDataObject, createSuccessfulRemoteDataObject$ } from '../../shared/remote-data.utils';
 import { PaginatedList } from '../data/paginated-list.model';
 import { Bitstream } from '../shared/bitstream.model';
@@ -96,6 +98,7 @@ describe('MetadataService', () => {
 
     appConfig = {
       item: {
+        noIndex: [],
         bitstream: {
           pageSize: 5
         }
@@ -406,6 +409,90 @@ describe('MetadataService', () => {
     });
   });
 
+  describe('robots meta tag', () => {
+    const noIndexTag = { name: 'robots', content: 'noindex, noarchive' };
+
+    const routeTo = (dso: any) => {
+      (metadataService as any).processRouteChange({
+        data: { value: { dso: createSuccessfulRemoteDataObject(dso) } }
+      });
+      tick();
+    };
+
+    it('should not add a robots tag for a normal discoverable item', fakeAsync(() => {
+      routeTo(ItemMock);
+      expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' }));
+    }));
+
+    it('should add a robots noindex tag for a non-discoverable item', fakeAsync(() => {
+      routeTo(NonDiscoverableItemMock);
+      expect(meta.addTag).toHaveBeenCalledWith(noIndexTag);
+    }));
+
+    it('should add a robots noindex tag when the item uuid is configured', fakeAsync(() => {
+      appConfig.item.noIndex = ['0ec7ff22-f211-40ab-a69e-c819b0b1f357'];
+      routeTo(ItemMock);
+      expect(meta.addTag).toHaveBeenCalledWith(noIndexTag);
+    }));
+
+    it('should add a robots noindex tag when the item handle is configured', fakeAsync(() => {
+      appConfig.item.noIndex = ['10673/6'];
+      routeTo(ItemMock);
+      expect(meta.addTag).toHaveBeenCalledWith(noIndexTag);
+    }));
+
+    it('should normalize handle URLs, casing and whitespace in item.noIndex', fakeAsync(() => {
+      appConfig.item.noIndex = ['  HTTP://hdl.handle.net/10673/6  '];
+      routeTo(ItemMock);
+      expect(meta.addTag).toHaveBeenCalledWith(noIndexTag);
+    }));
+
+    it('should not add a robots tag for an item that is not configured', fakeAsync(() => {
+      appConfig.item.noIndex = ['11025/9501', 'f4c45569-cdfc-4b3d-98df-46bfeba016b9'];
+      routeTo(ItemMock);
+      expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' }));
+    }));
+
+    it('should not add a robots tag for a non-Item DSpaceObject', fakeAsync(() => {
+      appConfig.item.noIndex = ['10673/6'];
+      routeTo(Object.assign(new DSpaceObject(), { uuid: '10673/6', handle: '10673/6', metadata: {} }));
+      expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' }));
+    }));
+
+    it('should register the robots tag in the meta tag store so it is cleared on the next route change', fakeAsync(() => {
+      appConfig.item.noIndex = ['10673/6'];
+      routeTo(ItemMock);
+      expect(store.dispatch).toHaveBeenCalledWith(new AddMetaTagAction('robots'));
+    }));
+
+    it('should suppress citation_pdf_url for a noindex item', fakeAsync(() => {
+      appConfig.item.noIndex = ['10673/6'];
+      routeTo(ItemMock);
+      expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_pdf_url' }));
+      expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_title' }));
+    }));
+
+    it('should keep citation_pdf_url for a normal item', fakeAsync(() => {
+      routeTo(ItemMock);
+      expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_pdf_url' }));
+    }));
+
+    it('should not break the other meta tags when item.noIndex is a scalar instead of a list', fakeAsync(() => {
+      // Must degrade to "off", never throw - that would strip the meta tags off every page.
+      appConfig.item.noIndex = '10673/6' as any;
+      expect(() => routeTo(ItemMock)).not.toThrow();
+      expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_title' }));
+      expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' }));
+    }));
+
+    it('should ignore non-string entries in item.noIndex without throwing', fakeAsync(() => {
+      appConfig.item.noIndex = [9501 as any, null, '10673/6'];
+      expect(() => routeTo(ItemMock)).not.toThrow();
+      expect(meta.addTag).toHaveBeenCalledWith(noIndexTag);
+      expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_title' }));
+    }));
+  });
+
   describe(`when there's no bitstream with an allowed format on the first page`, () => {
     let bitstreams;
 

src/app/core/metadata/metadata.service.ts

@@ -63,6 +63,9 @@ const tagsInUseSelector =
     (state: MetaTagState) => state.tagsInUse,
   );
 
+// No `nofollow`: crawlers should still follow the bitstream links to pick up their own noindex.
+export const NO_INDEX_META_CONTENT = 'noindex, noarchive';
+
 @Injectable()
 export class MetadataService {
 
@@ -147,6 +150,8 @@ export class MetadataService {
 
   private setDSOMetaTags(): void {
 
+    this.setNoIndexTag();
+
     this.setTitleTag();
     this.setDescriptionTag();
 
@@ -194,6 +199,46 @@ export class MetadataService {
 
   }
 
+  /**
+   * Add <meta name="robots"> for Items that must not be indexed by search engines.
+   *
+   * Uses addMetaTag() so the tag is registered in the meta tag store and cleared on the next route
+   * change; this.meta.addTag() would leak it onto every page visited afterwards.
+   */
+  protected setNoIndexTag(): void {
+    if (this.isNoIndex()) {
+      this.addMetaTag('robots', NO_INDEX_META_CONTENT);
+    }
+  }
+
+  private isNoIndex(): boolean {
+    if (!(this.currentObject.value instanceof Item)) {
+      return false;
+    }
+    const item = this.currentObject.value as Item;
+    if (item.isDiscoverable === false) {
+      return true;
+    }
+    // Array.isArray, not hasNoValue: a misconfigured scalar also has a length, and throwing here
+    // would strip the meta tags off every page.
+    const configured = this.appConfig?.item?.noIndex;
+    if (!Array.isArray(configured) || configured.length === 0) {
+      return false;
+    }
+    const itemIds = [item.uuid, item.handle]
+      .filter((id) => isNotEmpty(id))
+      .map((id) => this.normalizeNoIndexId(id));
+    return configured.some((id: any) => typeof id === 'string' && isNotEmpty(id)
+      && itemIds.includes(this.normalizeNoIndexId(id)));
+  }
+
+  // Accepts a bare handle, a hdl.handle.net URL or a uuid, in any casing.
+  private normalizeNoIndexId(id: string): string {
+    return id.trim().toLowerCase()
+      .replace(/^https?:\/\/hdl\.handle\.net\//, '')
+      .replace(/^\/+/, '');
+  }
+
   /**
    * Add <meta name="title" ... >  to the <head>
    */
@@ -349,6 +394,10 @@ export class MetadataService {
    * Add <meta name="citation_pdf_url" ... >  to the <head>
    */
   private setCitationPdfUrlTag(): void {
+    // Google Scholar keys off this tag and does not reliably honour the landing page robots tag.
+    if (this.isNoIndex()) {
+      return;
+    }
     if (this.currentObject.value instanceof Item) {
       const item = this.currentObject.value as Item;
 

src/app/shared/mocks/item.mock.ts

@@ -294,3 +294,12 @@ export const ItemMock: Item = Object.assign(new Item(), {
   )
 });
 /* eslint-enable @typescript-eslint/no-shadow */
+
+// `metadata` is copied because spec helpers mutate it in place and would hit ItemMock too.
+export const NonDiscoverableItemMock: Item = Object.assign(new Item(), ItemMock, {
+  handle: '10673/7',
+  id: '0ec7ff22-f211-40ab-a69e-c819b0b1f358',
+  uuid: '0ec7ff22-f211-40ab-a69e-c819b0b1f358',
+  isDiscoverable: false,
+  metadata: Object.assign({}, ItemMock.metadata),
+});

src/config/default-app-config.ts

@@ -281,6 +281,8 @@ export class DefaultAppConfig implements AppConfig {
     },
     // Show the item access status label in items lists
     showAccessStatuses: false,
+    // Item uuids or handles to exclude from search engine indexes
+    noIndex: [],
     bitstream: {
       // Number of entries in the bitstream list in the item view page.
       // Rounded to the nearest size in the list of selectable sizes on the

src/config/item-config.interface.ts

@@ -7,6 +7,9 @@ export interface ItemConfig extends Config {
   // This is used to show the access status label of items in results lists
   showAccessStatuses: boolean;
 
+  // Item uuids or handles to exclude from search engine indexes. Empty = off.
+  noIndex: string[];
+
   bitstream: {
     // Number of entries in the bitstream list in the item view page.
     // Rounded to the nearest size in the list of selectable sizes on the

src/environments/environment.test.ts

@@ -250,6 +250,8 @@ export const environment: BuildConfig = {
     },
     // Show the item access status label in items lists
     showAccessStatuses: false,
+    // Items excluded from search engine indexes (uuids or handles)
+    noIndex: [],
     bitstream: {
       // Number of entries in the bitstream list in the item view page.
       // Rounded to the nearest size in the list of selectable sizes on the