diff --git a/src/robots.txt.ejs b/src/robots.txt.ejs index e25e196d641..7aad1468635 100644 --- a/src/robots.txt.ejs +++ b/src/robots.txt.ejs @@ -5,7 +5,6 @@ Sitemap: <%= origin %>/sitemap_index.html ########################## # Default Access Group -# (NOTE: blank lines are not allowable in a group record) ########################## User-agent: * # Disable access to Discovery search and filters; admin pages; processes; submission; workspace; workflow & profile page @@ -26,11 +25,11 @@ Disallow: /communities/*?spc.page # Optionally uncomment the following line ONLY if sitemaps are working # and you have verified that your site is being indexed correctly. # Disallow: /browse/* -# + # If you have configured DSpace (Solr-based) Statistics to be publicly # accessible, then you may not want this content to be indexed # Disallow: /statistics -# + # You also may wish to disallow access to the following paths, in order # to stop web spiders from accessing user-based content # Disallow: /contact