From 218b17c7f51f172204fdc87954d1ffaf56a10d8d Mon Sep 17 00:00:00 2001 From: Alan Orth Date: Thu, 23 Jul 2026 15:52:26 +0300 Subject: [PATCH] src/robots.txt.ejs: remove blank line comments Remove comment about blank lines not being allowed. This may have been true at one point, but recent verification with Google's own robotstxt tool shows no difference in parsing with or without the blank lines. Also, the RFC for the Robots Exclusion Protocol does not mention such a limitation. See: https://github.com/google/robotstxt See: https://www.rfc-editor.org/rfc/rfc9309.html (cherry picked from commit a000038b841fda1fa3daf5771788244d5c24fe19) --- src/robots.txt.ejs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/robots.txt.ejs b/src/robots.txt.ejs index e25e196d641..7aad1468635 100644 --- a/src/robots.txt.ejs +++ b/src/robots.txt.ejs @@ -5,7 +5,6 @@ Sitemap: <%= origin %>/sitemap_index.html ########################## # Default Access Group -# (NOTE: blank lines are not allowable in a group record) ########################## User-agent: * # Disable access to Discovery search and filters; admin pages; processes; submission; workspace; workflow & profile page @@ -26,11 +25,11 @@ Disallow: /communities/*?spc.page # Optionally uncomment the following line ONLY if sitemaps are working # and you have verified that your site is being indexed correctly. # Disallow: /browse/* -# + # If you have configured DSpace (Solr-based) Statistics to be publicly # accessible, then you may not want this content to be indexed # Disallow: /statistics -# + # You also may wish to disallow access to the following paths, in order # to stop web spiders from accessing user-based content # Disallow: /contact