From 7623cd158f43e79f5f5e81793eec384c19543491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Wed, 18 Feb 2026 09:26:46 +0100 Subject: [PATCH 01/41] UFAL/Fixed failing integration test (ufal/clarin-dspace#1332) (#1249) * Add debug messages to fauling test (cherry picked from commit 4cc3694b1f75124f5d945f26e256a4b91f34d2d9) Co-authored-by: Milan Kuchtiak --- .../test/java/org/dspace/workflow/WorkflowCurationIT.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dspace-api/src/test/java/org/dspace/workflow/WorkflowCurationIT.java b/dspace-api/src/test/java/org/dspace/workflow/WorkflowCurationIT.java index 66dd2cee807f..dfe61a30b2b2 100644 --- a/dspace-api/src/test/java/org/dspace/workflow/WorkflowCurationIT.java +++ b/dspace-api/src/test/java/org/dspace/workflow/WorkflowCurationIT.java @@ -22,6 +22,7 @@ import org.dspace.content.Community; import org.dspace.content.MetadataValue; import org.dspace.content.service.ItemService; +import org.dspace.core.LegacyPluginServiceImpl; import org.dspace.ctask.testing.MarkerTask; import org.dspace.eperson.EPerson; import org.dspace.util.DSpaceConfigurationInitializer; @@ -29,6 +30,7 @@ import org.dspace.xmlworkflow.storedcomponents.XmlWorkflowItem; import org.junit.Test; import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; @@ -46,6 +48,8 @@ public class WorkflowCurationIT extends AbstractIntegrationTestWithDatabase { @Inject private ItemService itemService; + @Autowired + private LegacyPluginServiceImpl legacyPluginService; /** * Basic smoke test of a curation task attached to a workflow step. @@ -56,6 +60,7 @@ public class WorkflowCurationIT public void curationTest() throws Exception { context.turnOffAuthorisationSystem(); + legacyPluginService.clearNamedPluginClasses(); //** GIVEN ** From 78f9648a682622cd5428f469b60b99d7650aff57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Thu, 19 Feb 2026 09:51:08 +0100 Subject: [PATCH 02/41] [Port to dtq-dev] Fix OpenAIRE integration: null handling and HTTP client lifecycle (#1248) * Fix OpenAIRE integration: null handling and HTTP client lifecycle (ufal/clarin-dspace#1330) * Add test for OpenAIRE connector * Initial plan * Add null check for OpenAIRE response to prevent NullPointerException Co-authored-by: kosarko <1842385+kosarko@users.noreply.github.com> * Fix HTTP client lifecycle to prevent premature connection closure Co-authored-by: kosarko <1842385+kosarko@users.noreply.github.com> * Keep the try with resources but copy the response into an in memory stream and return that * license:check --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: kosarko <1842385+kosarko@users.noreply.github.com> (cherry picked from commit 02984dbe314ad80e192624cda7f8bc7c99a0eba8) * Handle NumberFormatException in OpenAIREFundingDataProvider.getNumberOfResults and use explicit UTF-8 charset in OpenAIRERestConnectorTest --------- Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> Co-authored-by: kosarko <1842385+kosarko@users.noreply.github.com> Co-authored-by: milanmajchrak --- checkstyle-suppressions.xml | 1 + .../external/OpenAIRERestConnector.java | 7 ++- .../impl/OpenAIREFundingDataProvider.java | 14 ++++- .../external/OpenAIRERestConnectorTest.java | 63 +++++++++++++++++++ .../impl/OpenAIREFundingDataProviderTest.java | 19 ++++++ 5 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 dspace-api/src/test/java/org/dspace/external/OpenAIRERestConnectorTest.java diff --git a/checkstyle-suppressions.xml b/checkstyle-suppressions.xml index 46bd9ca80d62..963165f75cef 100644 --- a/checkstyle-suppressions.xml +++ b/checkstyle-suppressions.xml @@ -8,4 +8,5 @@ on JMockIt Expectations blocks and similar. See https://github.com/checkstyle/checkstyle/issues/3739 --> + diff --git a/dspace-api/src/main/java/org/dspace/external/OpenAIRERestConnector.java b/dspace-api/src/main/java/org/dspace/external/OpenAIRERestConnector.java index 8b5fb1e523c5..7d1d40b03359 100644 --- a/dspace-api/src/main/java/org/dspace/external/OpenAIRERestConnector.java +++ b/dspace-api/src/main/java/org/dspace/external/OpenAIRERestConnector.java @@ -207,8 +207,11 @@ public InputStream get(String file, String accessToken) { break; } - // do not close this httpClient - result = getResponse.getEntity().getContent(); + // the client will be closed, we need to copy the response stream to a new one that we can return + try (InputStream is = getResponse.getEntity().getContent()) { + byte[] bytes = is.readAllBytes(); + result = new java.io.ByteArrayInputStream(bytes); + } } } catch (MalformedURLException e1) { getGotError(e1, url + '/' + file); diff --git a/dspace-api/src/main/java/org/dspace/external/provider/impl/OpenAIREFundingDataProvider.java b/dspace-api/src/main/java/org/dspace/external/provider/impl/OpenAIREFundingDataProvider.java index 8ca5b7c0ea5c..a46080698811 100644 --- a/dspace-api/src/main/java/org/dspace/external/provider/impl/OpenAIREFundingDataProvider.java +++ b/dspace-api/src/main/java/org/dspace/external/provider/impl/OpenAIREFundingDataProvider.java @@ -169,7 +169,19 @@ public int getNumberOfResults(String query) { String encodedQuery = encodeValue(query); Response projectResponse = connector.searchProjectByKeywords(0, 0, encodedQuery); - return Integer.parseInt(projectResponse.getHeader().getTotal()); + if (projectResponse == null || projectResponse.getHeader() == null) { + return 0; + } + String total = projectResponse.getHeader().getTotal(); + if (StringUtils.isBlank(total)) { + return 0; + } + try { + return Integer.parseInt(total); + } catch (NumberFormatException e) { + log.error("Failed to parse search result count from OpenAIRE: {}", e.getMessage()); + return 0; + } } /** diff --git a/dspace-api/src/test/java/org/dspace/external/OpenAIRERestConnectorTest.java b/dspace-api/src/test/java/org/dspace/external/OpenAIRERestConnectorTest.java new file mode 100644 index 000000000000..940ccb93fec6 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/external/OpenAIRERestConnectorTest.java @@ -0,0 +1,63 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.external; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import eu.openaire.jaxb.model.Response; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.dspace.app.client.DSpaceHttpClientFactory; +import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + + +public class OpenAIRERestConnectorTest { + + @Test + public void searchProjectByKeywords() { + try (InputStream is = this.getClass().getResourceAsStream("openaire-projects.xml"); + MockWebServer mockServer = new MockWebServer()) { + String projects = new String(is.readAllBytes(), StandardCharsets.UTF_8) + .replaceAll("( mushroom)", "( DEADBEEF)"); + mockServer.enqueue(new MockResponse().setResponseCode(200).setBody(projects)); + + // setup mocks so we don't have to set whole DSpace kernel etc. + // still, the idea is to test how the get method behaves + CloseableHttpClient httpClient = spy(HttpClientBuilder.create().build()); + doReturn(httpClient.execute(new HttpGet(mockServer.url("").toString()))) + .when(httpClient).execute(Mockito.any()); + + DSpaceHttpClientFactory mock = Mockito.mock(DSpaceHttpClientFactory.class); + when(mock.build()).thenReturn(httpClient); + + try (MockedStatic mockedFactory = + Mockito.mockStatic(DSpaceHttpClientFactory.class)) { + mockedFactory.when(DSpaceHttpClientFactory::getInstance).thenReturn(mock); + OpenAIRERestConnector connector = new OpenAIRERestConnector(mockServer.url("").toString()); + Response response = connector.searchProjectByKeywords(0, 10, "keyword"); + // Basically check it doesn't throw UnmarshallerException and that we are getting our mocked response + assertTrue("Expected the query to contain the replaced keyword", + response.getHeader().getQuery().contains("DEADBEEF")); + } + } catch (IOException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/dspace-api/src/test/java/org/dspace/external/provider/impl/OpenAIREFundingDataProviderTest.java b/dspace-api/src/test/java/org/dspace/external/provider/impl/OpenAIREFundingDataProviderTest.java index 5e96f06ac8ae..fb78313dec57 100644 --- a/dspace-api/src/test/java/org/dspace/external/provider/impl/OpenAIREFundingDataProviderTest.java +++ b/dspace-api/src/test/java/org/dspace/external/provider/impl/OpenAIREFundingDataProviderTest.java @@ -14,7 +14,9 @@ import java.util.List; import java.util.Optional; +import eu.openaire.jaxb.model.Response; import org.dspace.AbstractDSpaceTest; +import org.dspace.external.OpenAIRERestConnector; import org.dspace.external.factory.ExternalServiceFactory; import org.dspace.external.model.ExternalDataObject; import org.dspace.external.provider.ExternalDataProvider; @@ -102,4 +104,21 @@ public void testGetDataObjectWInvalidId() { assertTrue("openAIREFunding.getExternalDataObject.notExists:WRONGID", result.isEmpty()); } + + @Test + public void testGetNumberOfResultsWhenResponseIsNull() { + // Create a mock connector that returns null + OpenAIREFundingDataProvider provider = new OpenAIREFundingDataProvider(); + provider.setSourceIdentifier("test"); + provider.setConnector(new OpenAIRERestConnector("test") { + @Override + public Response searchProjectByKeywords(int page, int size, String... keywords) { + return null; + } + }); + + // Should return 0 when response is null, not throw NullPointerException + int result = provider.getNumberOfResults("test"); + assertEquals("Should return 0 when response is null", 0, result); + } } From 98edc1d737a0cbcce302923a82c395015073df91 Mon Sep 17 00:00:00 2001 From: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> Date: Thu, 19 Feb 2026 12:02:12 +0100 Subject: [PATCH 03/41] UFAL/Added a comment to do not forget mounting the file which is changed via ocnfiguration feature (#1247) --- dspace/config/dspace.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index 9599647369a1..8330d7ee3cf1 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -1697,4 +1697,5 @@ include = ${module_dir}/external-providers.cfg # Configuration files that can be updated via the admin API # Comma-separated list of file names relative to ${dspace.dir}/config directory # Only these files will be allowed for reading and updating through the REST API +# NOTE! This file should be mounted because after restarting the backend those changes will be lost config.admin.updateable.files = item-submission.xml \ No newline at end of file From 475b25aa88ed756c09804530f47880ea291e8e5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Thu, 19 Feb 2026 12:54:50 +0100 Subject: [PATCH 04/41] UFAL/Issue 1315: Store file preview to database when file preview is created on Item Page load. (ufal/clarin-dspace#1316) (#1241) * Issue ufal/clarin-dspace1315: Store file preview to database when file preview is created on item page load * assert text improvement * PR comments: commit context only when any of the file preview is successfully created * change variable name (cherry picked from commit aab626b39ffff1da38f65aa5dffa0ad834856eeb) Co-authored-by: Milan Kuchtiak --- .../MetadataBitstreamRestRepository.java | 11 ++++++-- .../MetadataBitstreamRestRepositoryIT.java | 27 ++++++++++--------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataBitstreamRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataBitstreamRestRepository.java index 7fea50431b3a..008635fbeeee 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataBitstreamRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataBitstreamRestRepository.java @@ -66,8 +66,7 @@ public class MetadataBitstreamRestRepository extends DSpaceRestRepository findByHandle(@Parameter(value = "handle", required = true) String handle, @Parameter(value = "fileGrpType") String fileGrpType, - Pageable pageable) - throws Exception { + Pageable pageable) throws Exception { if (StringUtils.isBlank(handle)) { throw new DSpaceBadRequestException("handle cannot be null!"); } @@ -80,6 +79,8 @@ public Page findByHandle(@Parameter(value = "handl List rs = new ArrayList<>(); DSpaceObject dso; + boolean previewContentCreated = false; + try { dso = handleService.resolveToObject(context, handle); } catch (Exception e) { @@ -127,6 +128,7 @@ public Page findByHandle(@Parameter(value = "handl for (FileInfo fi : fileInfos) { previewContentService.createPreviewContent(context, bitstream, fi); } + previewContentCreated = true; } } } else { @@ -147,6 +149,11 @@ public Page findByHandle(@Parameter(value = "handl } } + // commit changes if any preview content was generated + if (previewContentCreated) { + context.commit(); + } + return new PageImpl<>(rs, pageable, rs.size()); } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamRestRepositoryIT.java index d28c814f4468..544240beffcc 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamRestRepositoryIT.java @@ -12,6 +12,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; @@ -84,16 +85,16 @@ public void setup() throws Exception { .build(); // create empty THUMBNAIL bundle - bundleService.create(context, publicItem, "THUMBNAIL"); - - String bitstreamContent = "ThisIsSomeDummyText"; - InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8); - bts = BitstreamBuilder. - createBitstream(context, publicItem, is) - .withName("Bitstream") - .withDescription("Description") - .withMimeType("application/x-gzip") - .build(); + bundleService.create(context, publicItem, "ORIGINAL"); + + try (InputStream is = getClass().getResourceAsStream("assetstore/logos.tgz")) { + bts = BitstreamBuilder. + createBitstream(context, publicItem, is) + .withName("Bitstream") + .withDescription("Description") + .withMimeType("application/x-gtar") + .build(); + } // Allow composing of file preview in the config configurationService.setProperty("create.file-preview.on-item-page-load", true); @@ -116,6 +117,8 @@ public void findByHandle() throws Exception { // There is no restriction, so the user could preview the file boolean canPreview = true; + assertFalse("Expects preview content not created yet.", previewContentService.hasPreview(context, bts)); + getClient().perform(get(METADATABITSTREAM_SEARCH_BY_HANDLE_ENDPOINT) .param("handle", publicItem.getHandle()) .param("fileGrpType", FILE_GRP_TYPE)) @@ -135,13 +138,13 @@ public void findByHandle() throws Exception { .value(hasItem(is((int) bts.getSizeBytes())))) .andExpect(jsonPath("$._embedded.metadatabitstreams[*].canPreview") .value(Matchers.containsInAnyOrder(Matchers.is(canPreview)))) - .andExpect(jsonPath("$._embedded.metadatabitstreams[*].fileInfo").exists()) + .andExpect(jsonPath("$._embedded.metadatabitstreams[0].fileInfo").value(Matchers.hasSize(2))) .andExpect(jsonPath("$._embedded.metadatabitstreams[*].checksum") .value(Matchers.containsInAnyOrder(Matchers.containsString(bts.getChecksum())))) .andExpect(jsonPath("$._embedded.metadatabitstreams[*].href") .value(Matchers.containsInAnyOrder(Matchers.containsString(url)))); - + assertTrue("Expects preview content created and stored.", previewContentService.hasPreview(context, bts)); } @Test From ff0427ebebe9abecddad985c50ca31f2fdb0680c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Thu, 19 Feb 2026 13:01:14 +0100 Subject: [PATCH 05/41] UFAL/Issue 1313: fixed error when file preview is not generated for bitstream with store_number = 77 (ufal/clarin-dspace#1318) (#1240) * Issue ufal/clarin-dspace#1313: fixed error when file preview is not generated for bitstream with store number = 77 * resolve MR comments (cherry picked from commit 04d64f718f3964ed2174ca36d0ade324fe20e997) Co-authored-by: Milan Kuchtiak --- .../SyncBitstreamStorageServiceImpl.java | 10 +- .../dspace/builder/WorkspaceItemBuilder.java | 23 +++++ .../scripts/filepreview/FilePreviewIT.java | 87 ++++++++++++++---- .../org/dspace/scripts/filepreview/logos.tgz | Bin 0 -> 18980 bytes 4 files changed, 101 insertions(+), 19 deletions(-) create mode 100644 dspace-api/src/test/resources/org/dspace/scripts/filepreview/logos.tgz diff --git a/dspace-api/src/main/java/org/dspace/storage/bitstore/SyncBitstreamStorageServiceImpl.java b/dspace-api/src/main/java/org/dspace/storage/bitstore/SyncBitstreamStorageServiceImpl.java index d2266f02d75c..2ea0ffe6aaa4 100644 --- a/dspace-api/src/main/java/org/dspace/storage/bitstore/SyncBitstreamStorageServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/storage/bitstore/SyncBitstreamStorageServiceImpl.java @@ -7,6 +7,7 @@ */ package org.dspace.storage.bitstore; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.sql.SQLException; @@ -185,12 +186,17 @@ public Map computeChecksumSpecStore(Context context, Bitstream bitstream, int st } @Override - public InputStream retrieve(Context context, Bitstream bitstream) - throws SQLException, IOException { + public InputStream retrieve(Context context, Bitstream bitstream) throws SQLException, IOException { int storeNumber = this.whichStoreNumber(bitstream); return this.getStore(storeNumber).get(bitstream); } + @Override + public File retrieveFile(Context context, Bitstream bitstream) throws IOException { + int storeNumber = whichStoreNumber(bitstream); + return this.getStore(storeNumber).getFile(bitstream); + } + @Override public void cleanup(boolean deleteDbRecords, boolean verbose) throws SQLException, IOException, AuthorizeException { Context context = new Context(Context.Mode.BATCH_EDIT); diff --git a/dspace-api/src/test/java/org/dspace/builder/WorkspaceItemBuilder.java b/dspace-api/src/test/java/org/dspace/builder/WorkspaceItemBuilder.java index 580e4dfef61f..75c8ea886a2a 100644 --- a/dspace-api/src/test/java/org/dspace/builder/WorkspaceItemBuilder.java +++ b/dspace-api/src/test/java/org/dspace/builder/WorkspaceItemBuilder.java @@ -249,6 +249,29 @@ public WorkspaceItemBuilder withFulltext(String name, String source, InputStream return this; } + /** + * Add bitstream with specific store number. + * + * @param name bitstream name + * @param source bitstream test source location + * @param is input stream of the bitstream + * @param storeNumber store number + * + * @return this WorkspaceItemBuilder + */ + public WorkspaceItemBuilder withBitstream(String name, String source, InputStream is, int storeNumber) { + try { + Item item = workspaceItem.getItem(); + Bitstream b = itemService.createSingleBitstream(context, is, item); + b.setStoreNumber(storeNumber); + b.setName(context, name); + b.setSource(context, source); + } catch (Exception e) { + handleException(e); + } + return this; + } + /** * Create workspaceItem with any metadata * @param schema metadataSchema name e.g. `dc` diff --git a/dspace-api/src/test/java/org/dspace/scripts/filepreview/FilePreviewIT.java b/dspace-api/src/test/java/org/dspace/scripts/filepreview/FilePreviewIT.java index 7d3cad26fc0a..d03384c25d7b 100644 --- a/dspace-api/src/test/java/org/dspace/scripts/filepreview/FilePreviewIT.java +++ b/dspace-api/src/test/java/org/dspace/scripts/filepreview/FilePreviewIT.java @@ -13,6 +13,8 @@ import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import java.io.InputStream; import java.sql.SQLException; @@ -36,7 +38,11 @@ import org.dspace.content.factory.ContentServiceFactory; import org.dspace.content.service.BitstreamFormatService; import org.dspace.content.service.BitstreamService; +import org.dspace.content.service.PreviewContentService; import org.dspace.eperson.EPerson; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; +import org.dspace.storage.bitstore.SyncBitstreamStorageServiceImpl; import org.junit.Before; import org.junit.Test; @@ -45,10 +51,14 @@ * @author Milan Majchrak (milan.majchrak at dataquest.sk) */ public class FilePreviewIT extends AbstractIntegrationTestWithDatabase { - BitstreamService bitstreamService = ContentServiceFactory.getInstance().getBitstreamService(); + private static final int SYNC_STORE_NUMBER = SyncBitstreamStorageServiceImpl.SYNCHRONIZED_STORES_NUMBER; + BitstreamService bitstreamService = ContentServiceFactory.getInstance().getBitstreamService(); BitstreamFormatService bitstreamFormatService = ContentServiceFactory.getInstance().getBitstreamFormatService(); + PreviewContentService previewContentService = ContentServiceFactory.getInstance().getPreviewContentService(); + ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); + Collection collection; Item item; EPerson eperson; String PASSWORD = "test"; @@ -61,7 +71,7 @@ public void setup() throws SQLException, AuthorizeException { eperson = EPersonBuilder.createEPerson(context) .withEmail("test@test.edu").withPassword(PASSWORD).build(); Community community = CommunityBuilder.createCommunity(context).withName("Com").build(); - Collection collection = CollectionBuilder.createCollection(context, community).withName("Col").build(); + collection = CollectionBuilder.createCollection(context, community).withName("Col").build(); WorkspaceItem wItem = WorkspaceItemBuilder.createWorkspaceItem(context, collection) .withFulltext("preview-file-test.zip", "/local/path/preview-file-test.zip", previewZipIs) .build(); @@ -116,22 +126,45 @@ public void testWhenNoFilesRun() throws Exception { @Test public void testForSpecificItem() throws Exception { // Run the script - TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "file-preview", "-u", item.getID().toString(), - "-e", eperson.getEmail(), "-p", PASSWORD}; - int run = ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), - testDSpaceRunnableHandler, kernelImpl); - assertEquals(0, run); - // There should be no errors or warnings - checkNoError(testDSpaceRunnableHandler); + runScriptForItemWithBitstreams(item); + } - // There should be an info message about generating the file previews for the specified item - List messages = testDSpaceRunnableHandler.getInfoMessages(); - assertThat(messages, hasSize(2)); - assertThat(messages, hasItem(containsString("Generate the file previews for the specified item with " + - "the given UUID: " + item.getID()))); - assertThat(messages, - hasItem(containsString("Authentication by user: " + eperson.getEmail()))); + @Test + public void testPreviewWithSyncStorage() throws Exception { + configurationService.setProperty("sync.storage.service.enabled", true); + + context.turnOffAuthorisationSystem(); + + WorkspaceItem wItem2; + try (InputStream tgzFile = getClass().getResourceAsStream("logos.tgz")) { + wItem2 = WorkspaceItemBuilder.createWorkspaceItem(context, collection) + .withBitstream("logos.tgz", "/local/path/logos.tgz", tgzFile, SYNC_STORE_NUMBER) + .build(); + } + + context.restoreAuthSystemState(); + + // Get the item and its bitstream + Item item2 = wItem2.getItem(); + List bundles = item2.getBundles(); + Bitstream bitstream2 = bundles.get(0).getBitstreams().get(0); + + // Set the bitstream format to application/zip + BitstreamFormat bitstreamFormat = bitstreamFormatService.findByMIMEType(context, "application/x-gtar"); + bitstream2.setFormat(context, bitstreamFormat); + bitstreamService.update(context, bitstream2); + context.commit(); + context.reloadEntity(bitstream2); + context.reloadEntity(item2); + + runScriptForItemWithBitstreams(item2); + + Bitstream b2 = bitstreamService.findAll(context).stream() + .filter(b -> b.getStoreNumber() == SYNC_STORE_NUMBER) + .findFirst().orElse(null); + + assertNotNull(b2); + assertTrue("Expects preview content created and stored.", previewContentService.hasPreview(context, b2)); } @Test @@ -150,4 +183,24 @@ private void checkNoError(TestDSpaceRunnableHandler testDSpaceRunnableHandler) { assertThat(testDSpaceRunnableHandler.getErrorMessages(), empty()); assertThat(testDSpaceRunnableHandler.getWarningMessages(), empty()); } + + private void runScriptForItemWithBitstreams(Item item) throws Exception { + // Run the script + TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "file-preview", "-u", item.getID().toString(), + "-e", eperson.getEmail(), "-p", PASSWORD}; + int run = ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), + testDSpaceRunnableHandler, kernelImpl); + assertEquals(0, run); + // There should be no errors or warnings + checkNoError(testDSpaceRunnableHandler); + + // There should be an info message about generating the file previews for the specified item + List messages = testDSpaceRunnableHandler.getInfoMessages(); + assertThat(messages, hasSize(2)); + assertThat(messages, hasItem(containsString("Generate the file previews for the specified item with " + + "the given UUID: " + item.getID()))); + assertThat(messages, + hasItem(containsString("Authentication by user: " + eperson.getEmail()))); + } } diff --git a/dspace-api/src/test/resources/org/dspace/scripts/filepreview/logos.tgz b/dspace-api/src/test/resources/org/dspace/scripts/filepreview/logos.tgz new file mode 100644 index 0000000000000000000000000000000000000000..ad1e02dc605973cb27944b5002a12952163b0a5d GIT binary patch literal 18980 zcmV(zK<2+6iwFQ5z6fam1MItXcpXcUCo0KeW{WLmI7dg!%*@Qp%q)vpvY45fEQ>5= zX13U3i)G>Z_HCHy?Y^_SZ+hPM-cHnaPF7__WJY90MAjd_P%!k6Ff}wWbz(Acb}%$H zWw5a~w`X*)GY9+6BLD!v#==7UTg%SI`llAa{QL8th#A023}giYn1LWxAUiuT0LTtt zWdS4h_)pZ~pOUMyi=opmRcx(YjV)X(4XytP-!C>Zvp*Q;cTQS>RdOiDe{(1eJs(B-KP2Jq%r3oEVMmZ5a(69BfP(9h~gl zOzjNqj7@n36(t3Q9p#)v6j|LlU0L8+IEj^i3CU>wkq|fp7<{2M60*bHIsE_Ii|~Ij z{p+IsZ}i9U&;08p@bBWE6$k`^5d;1u+WZ~;UmpJv3UcB|@OZz4Ad;knh|;gt?zbN- z^soOxZOW%#4Y;$CxDZ&)B*D?I2RH`_O=mDL1dQK);9wb9IKLP)EmbsJG-PGC{uJsa z_C}_R9<~mDnt;cH>({NVsf!`8hpmmBGnWT1>7O{be%=4p%|uH4Cl(iLUQ!KN1!7@) zCsSfpMixc@DIYvBF)@#mi5ZuYh}fUSe?9S%TDZ74a4|8tySp>GgBb0d%$b0koSaMm zW+rB4hF=^E&YpHIh8_%d&Sbw2@|ST$Or4FLEFD}d?d^zvdp0z(cXi<E()KhgQNq9W2^dx0pnfk zolh_R8|P!|-0s+D1Rk~4T{&yIW91cZ^PH{~Un;Z`Wz0zp{ueKYPq4${>E~&bbhI=L zI$R#9vxEh?FZTy&TTJB};dK0$n-Py|lj;Iw*J@&U~<2 z0oNySq?m>n7$~maod_r_-ZG8`1^ir)Htt%xXW8mw3lmLjr$y>LL4j&Me(8aWvS0oj zO54H$r4yXr=+A5Qqiy%e@AyW8f)!rRML=7Xm4+Z(GNG;n8m}+2SMYWvWXaA0Ie<+8 zpI(c6#1emahSV>5)>R?fSfukEGbE(Uteww~swzcu(+d~#<*Pmt*68o$4JjhvAu*&i zjV;VS7N-VNb<5@qvqZX0rn`=EK+t~8bz5pH1)nKoql%3QzXJ(Jg*+EA!^R@ZGUiut z%B!l8H+=7J?Rn*P#K?AEB6hf-jyU#8o{`r*%=c6q`|w345V4G;roq{sX_nUvhq8#2*MBF5`Q`YQ13Vg|f& zkJBCBR(oTvKM=4!h#2d|F~>T%t<<_=Tu&AdLOEj<)i<$;l5+ZHnIjP;-#l|_41yF{FN8De6_!OeasAmPx2 zv9wkH4&>J=?eKBFzLT31cS-D{5*o{+yl5m%gXq78aZa;j+H zauO~faC&nUGv?sVS^uhX76BX8FZ$N2qq~u4&CS85JVBC$lm(YQkk;7FFXP1VBt8yZ z#=U28wW4ITVq>!%buJXL$qzDaMM+S&aRtlTtgDbII$@@Zz#DvGT|yy-M2)LN166)> z)SdCm*-WGQ6BF%IIIVIjU&2M|%SDk9!Pf+1MO7yb`k$@*{`u~gO-Nz0V3aubR~Ps5 z-SlUgX_Pz*W7|w^Ke+6>&iwRKUNEDNI9w!5iLMq3a6OU5F9bLBg9IFD6PTXbhH$AA z&i$%QSxd@uY%<#q6iyt-*I1oQ(8$YK7@&L~X>Dlz^9xR|0G@&@M->mC7As>bu*yIa;B**U^+&{E-OTEjZC%SxVK=dBUA(wvNW4g~dqZf5UUZI6886dVa zR_Rt86odlH_QqOdUoI18-Y0UtCPF;D4-k7dM=H+)%m}{x9=IXf+Jm#bPEElmNm3W6 zX%h+AcL0%~Z)g|?i1ECK-6aO#nUgM~^sINO_xME2SvR)%s7W91-)I@4(`LQoJ(4Qm z_)7yz*Jy8vzWG8-rN8dj8g}&7!v%vMBRDwA?GG;?S>~|YI~R%2H9Q+`9=H?R)^Z7a zU{S&?G;m-XAYGN%JHZ-+yFLP6B1bnXEi>MdBu-$Q3^Bt{%jVPTMYwXsm3$9k%& z;$@|ue4lXHCRRyv8TXMkmUS3AXK+>ip#5WJsN7|Z&)vLhCcM#LrEMTs%(Oea zoP(Kh=CdMoj6LRj8Xjy5web7sH*Zt&Se`zypz88cSK1zXt8IqP&x7md@}4y*CKpwYX4^v2h$ zQ<&*QJd1FiQzow`#qMqlX$M6f>j5hSYb_SAms5yV2~tf^PiYt1u>|e+)aU8lXk*`> zatxhzolZn8Im(T~l7#*Xfng9En< z`L4*Ry;pM<+a2LJ?)CccP{t7mOk%1MMwrL4=e|~h0*gStyWNeE6FJz36Gk7CJeHbh zQ?*QMVgK<_XPw9=vK=fc5rmuYVA{~I5Z)A8Ln(wBj3d6pbI5LPTyjP@A%DYku(!?f zH=n{bMjhm_L{|ycwWL|tY#2LQtE>@G$$YBAKFJ{tfm+87sL%K|a$}{~U7(8~N4Q0D zStAM``O(azIL81_Z%v;J1ll@fNp2d>*$JWC=;)+dP6>AaD&cUOPg!qLJ<*1V;-dwX zZV0fQWc2x(KL&UAZbf|7%H%$|%ga$eU#3SlI$PA;9FHnH-_@sgk94EqhuNzAfC*C@ zNm|e8SK`gqb^!5GsUQF|3>1S=@ME7N{W=_{&(KB}uMa&)Gu`vfTBurwIp|V5%QbCw zC>X2I0bh;bNAC{L04ZIX=JtAtiuRn+z>ZVy*Hq=x-PAESVC?Rv_Y~z4J359H*UqDw zyYO_eU;S=pp{||1+KGA!oPh74LZ^q-*ZDG8r!fb@GYvHXam)eN%v zVJRE+witU!7>UxbpN}u)-M%ztF{D8!1(RqtYO9*gtsY=YT2)aj z+dvWqbxnWJZ`z3ukJ8>lJ=AkTM|Es7F(&2aJeBr3RvaAA?$&=9!6@v`VW0N4<-i=1 zg1}B1o(WGmnP_?cxnpq~$HuuCSR7K2$5pRXx$R(a5h#yu_Pl*?zx5rl@tppAMeR$B zOL4_j3J4m@BykUdQ*WYL z>zxWF3TAaKak7e!@6*aL+${%z4^cwR@fPw3!CJ0oGyw7$1P0$a{a*ig@F|9gf&(;HaZl^Z(OYT^GRJvB3v3MJX z)L_9vCN}m>4|2M4y-L98_7ekJ(H`AoRm;9geYmPsOo%FGNWMhpf^s~`#iu2ANf*Tx(KCFl)HcFQSYA>`v6;3bGSDj*GZrzKR9H6y z(l!=ik=H44B4Vhgkl|!y`@>sE;AxXNtK*4oJhT;i6xY5ZIk!8 zfw--!O8KNI*d{K*{ntkeYrv}XTC1x9H_})rJp)og-UP!-lwR)L{WwTD3UJdBIin**ufdh_rQ(NW4*dzFeCP|Ej%A7+NmWkt z#ZA@su;#(lVc#1%Muw6{?Ao*&4*g^KIO+Anat2pY3d~ppT__7bt$07Lf$S|N$MRro zgf*`=o)ii>6znVqMbGwg_>+?c0(7_DUZWUGw5nssBe%qoq!^?0ytA)VviD3muXmJv zkMn(Adxu3vS%LBOfx%Pd^xH<-0AwTVD5Gr~44hrTDr(X}O*_#Z+Ie&64F%ZMJETA82Yy;9xSb1kH58Pq4^oP7L>26fpc)wTg;R=?fS ztf4$2QYSkW>bREQOy}l@AE+odwViraEUs%F@~Q+s=vj#|IdJ%;Ktt8mnuDNf`PQ%R zqbnWlz2kDc*Ql_3a;Laz@jGE~M#jSqGBAtMQ6)h`NTZ7DFJlkoN9~s~SY^Fn4Musk z)#Zp)FSX*Yyt~=z$f^v$Sv;aU~(K~MqGf$)Vn z7Aup{VS#wkP)(m*&0}rTu$f~G5+^%_kwgAn*n2&=5hFFomvk(990;uanAbHXZGX)m z%bhckCwi~ZQgIxZIuZ`Q{gGT7zV%sDUsUYOk1ME6PNLft)h_S*z$&uGrmRs~_$0nk zLp?|;wG388t5(G&ughiW_TD>MS{fzdWs1{ta1TUcWxzwvqt&lurc zlgmn_*0{xK2Y*)!oRFY|_}{zUmQcSvmDX$u`fjs-_3PGOX<_KPugc2Ag~TzDvNNdHtZ?gQi3uvw*g({%fp_LJG<@^;IlUEhS%=rYKqk3|fK zeZ$jx>#^q%E;k*qv0vZIENH%FhxjJgz*TU31%<%AtSB8jP_&F+ojJ2FVMSg;*KZ0p zeN!wHi{p5?A?ZvLw>`@rY-nh3T8|{mh9qE6rl&IZz&i}m$F6ZgLyR-~Bezypr3b%jbSRfLvF?1?EQZWA_@Qf+id8j#KdRLB8NtVR!$6OR zl~g|4OIXp&Y>`kS-U08sX&%d-3EiYn8FHAAKfi;)#XIZ8DoO8#%ARRUkX_dCbvT(% zTUcf(RX-inXU|~eOO;ckhoYLhA0MQoUT@evW++Bc{dumo^1S=THM_4r^ERR>p_sLk zZ4aMcT@j|{Db@?RroF;aFgAi&SY)Z|``(s4>-oN4wrz3LKu$y9hw1$cj*-ip#9vlj z-qu>Q7vk!*s1{m$(}=xLH*4G?lO=! zbS_^9z4a<6eqQDMs?w$9wfrlG{q>1X>2{od1vydugn#FPb)(rt8xuFx6EYbr6^Q~V z`up5>fr|>##6P0;rE~qjE~DYY$wetB;?H-Mt1=HJPc|x-b^`^#=sN2{W5tsJCFc&~ zH5%HItIe*>-I}lCiBMv=(HypZF+2_SKhN)fY3c1HPHI@MGj3Eb0Yi(d)9JMDT#GYq zo5Xm#m`5g?>$n>vqd&+impWEq6oEG;QEu+}Fz!uHm2`Y^{JvVFe!dn%s%WUu(LSB~ z{$N6Lh0{D#h$LbkEv@*Vihp<9*ryH7oX5$Uk!|opFWfh8$1`i&Sfa07eP+gQu4XmY zr#)nidQUU6&QwX3k2@LU4OIZ`Z}YgIn#s}0FTp2U8s8TM@^+^4g=-fV7rR1*R7L9T z0YPU#qPQtTWqR`jSh|cosap)3d5vgCa3RPh;a1GFen<2KGV z{!AXv<7WDIV~HXtC_gZKkE3xLDLH?f!4wTB7IUQIYXWMRK)cYdvjIEy;*-=Tvd#xD zUGBA;QCNB7%vV;HcCTf}-+H>|7nZ6~8Xa~pptZ=E`0cmH)7f7VS@M}js@#yT&=yZB zLi-ZF7yz2sG3QV4g|a=5T+Ky4D~tBA&w=IeoV3_i;j6rlMXs$JChB-Bm+#hUtMtr& zpjQW!T+Rpk1$W5nKkb5`Bbm4jlU3H%W8im#+1S4`$b(N`@Vms!mTcXJ_*DB_-jokg$hWqcr$9xIcHHbTfmuXnnF5I?IMPP}L!Y zAZvCN+s)$hHFH!mTeLNhgY-$d^sY1VANV-*!GCYuJXhCkW41Jx{@KL*C+9Tg8Ub=? z%+D3V7Mi?d*~bT4eZ!pZ0g=mRZpB^s=dWQOyc`sql{e}X%1kY&%k(>jOtC^{*HX)p1L;HlVolU;wq8?2srSL%Jj5>!dqO^@eYqT&kYgJn zfu~>%-T7=_!;I4r_ClEHl7akhZAC>tuEw^SMNHT`NFgQV@#7M%wk%%f1-yedB`kSz z0fHY5>)c+SFL=QnuoccUs~6|u*8v`|u?Yl+sbf}#+orL_w^w&fY@nk^T+EJ8{2+}KI&s@?~LTa`zEqs=YDy*;;c52 z#cRIGx`e67kf^D&lELczdKtH{2IprkW4zvSo)Y9}qtJ_s{&AxiLS|!)I_?$cZDx!T ziHN5T$7)uXCUkW3b_ZdqdRpa{H6=Cr#;bEKtv7EYC0uZKCyI#=<+3|etXwHEC~#5K z1h01>j7}g^1GfC-VfUUc*Xy``>}%6jP)bizi^<%kxaz_KEcD&IWK+PJUhL<7TuG?7 zdJ;YUm@K8kRG*-9tX!6^NY=YXk2)&aRr}GPgwmIz*RwTL@l!AgxWZ&;ZEbb42Osr5 zghP8v#j;vD6VbNbeTR<+TUVXDb>H-??d&waIPEuH)I(vQs?Af29oq+CaL^x+zKP<( zn$z{Gp>Zmyu$Uq~rg`QoC^W3tpsQTleH?JHWWgC-oLU`;A3hl-W~N0&tP1=PO3hkH zAa_{($zmE0k0eHceH>3hCMJ|!ZwIJ@K*DG1xWs#R%i}?prAI|_%f+x* z?<^@S9I%?)(Adz8B70>^Ew5!viX_XevDcE4a`0{>dJZJ>N`nzVv`+Y7Gn}3=Dm*d0 z?a}zi?Hk=%Js}=345 z&t@+Ol`qwn#IE;s?|D~VCeBAbo%Lh>cOfdC|2uOHkT5rLj-EC$iTR!51YemcHokC`}q^y0a#A6~$paPSP8x|Sb} zIomwvYTEs-8ZU1ML9(S&*xNr$O?)WGK*lO!VuRGtXyOIH08ciX&IU~@Wc4?@{cAcs zqg_jcPlha>zMDV8#p|#>ZL_AmnsS|`NYGzVFAzN&iY%ArI1b|ay5a?CMlaRs^DDAX zzh9(&nf6$IC?DiAGT#~?rpa>}WDXJ= zb77BhEBD@DXy`y)hWEk}Azx>Hmm8pVFf@b2gGgav)kjcA_4sxkQx(kJ!L@PWg6Y$f zoK50kc4I~^+hrTi;>yDV;bgD~rtB(scO)|O&R1CpdA=Q$Mw!}bYzm{`C zjC2fC3xa`2s**C2c01u|SCMW99rI`Dx|iYruy5enI|HdY&}M~i3YdtTJ0b=Kx^XV@ zj5?p=ed~x};4n7^xi308EoY1D7YvQa*7n_`(=|OC(YC9X^!<6EtrOEKst}L}K8Z6h z=G?_3CLZ#OcmXHE9I%yGAf8*Cs~}2(9d>dt@t9494fmL?Z1I7U94+`FYE_YSUS0fx z0E7wIL^1tU`>(D3d^T{%lKAjr%qFcG1YvaVfA^8~qNQroi#_3q$j>m*_m^y}*KY~6^VD@z|WPW)2)5=ypP z7(ODIMm18tnjQ5X4vj4dhAqZ^mg_EkSd$EVfe?IAAnBsvK4PKC=qFs`Dz=lKIETP4 z3Ie!`*ot~72ejM6qC5lY5Tq?+#V(=z;4e+tTy_4ypw|#-2?jrm^F9>zEzZZRTK(-y zkH_Gd?W)sp?}Le_^Zl3@q^X{G#yAlboVk@PH*cO#>lSiycM{K&;BzVD;YDqRz6Di6wx?`ep{Z0}YpJ3r>(+3Y;zA5E15R1|va(PAq)8BXlF zjd&M=itJZpQ6b~!54GN2IbRIik&VFfYFRZb^|zDLlW8L14EmXRRV)~xpvRN!t909Q zqrWESQIT$c_^u$=PZn8(jK?C;Z>ZN1?Q!HE^-Y(SGrPswpnv*DcwYVT-6?Ii*FuG= zi3E4p_9%A$K8|d|+Ue7tZ@CRqffoAn!|kV zcMVSh9;)P@XqrA2u+dSLX$G$rZ`~({>wZ5U?MY>mwwOqEJt(17F~tx$mTG)mPZs@s zIVxIe?4W2-4JkChA%>z3=n0R>)Iab&56>3MzkZsM$kh>gnYY0ll9P@SBhJFkR&a)bC1P;Iy0 z71qrXgYahL3I?uOcvotAu7s~qGwfR?E@+qJQrAnHK}F$Sa^2BB6;cB?Z&@56nD=yzw}a4-iEus z#FH=a-BoP1v`9mbEGBOa2aX|$_HOrjj$=tPrfoWWz=bdjQ(<3h9rF5Ht9;@jmagZ1 z>*Vmr$Y&l9$Qu$29rW4hl*(R$XHqWVW2$Y>D~P5%I*~ZK_6?+;@tGnUf5U z*8+B)Ibcn@5>54Cl|gmR@6+({F+>rQwLZe!kjU5n#p{~}$&c<2JQHLREM!H$;U*+S21|5QNe ze}Ml6va^Ezy8j?%X5wE50tocq`TxI!K!Cr)e{-@h0@*qL_K6wC4q8QqAqgQV&bx1Ho~$ZHX=%*vL5i9tpDuz_%|~Gm^pylY@7fNBSRBb zZea^2ds|a(HWqe}7`rGd11AV5$iN~31TqM*vH=;`1jRs{LI6%-5na-;g{hON*e^qzc{S~XoDDVP0J0*&mV$~>W}+f;_9hZa?(&u# zZbssqR+{P__C_E%fV`EWyR3+yo2G`GmzIW-mkhIwi>A7Xjj<=Em65oZr=dEF%P)RA zBTrU$BU=+QQ6Yyv9LG?c(?v_sTt-n*@XsVFB=}d775tw`$P^yP!Sc@*<_}CXa%XJ=CvXC_NqLvvH-zt2tjsCfTa%Ho$R zz#<6#?&09ze|GhEf9Lvo@IVmg54XqA*oIe7R@BNvOo+}D$SEePB}V6A!wHbK0Ex4eGnl@_PTDMfyMf#`<&q{}*KaztMky{|B*v zfd4-K1KB`q%>Rx5|F58bfd4lH`wRY`UBTDwSA**!s^Ow+Z|34|=wu2e_!|zMSn{`s zRQmPF;ORJG%JVC@$p4@OfZxNx(Ae74h1kf{+|rJh^rF3+l-SaQmsFiq79i^&Y-(XC z;pJqi>?NmS>}74tX+p}+NBqB4IPkkt|E7EZ#9#~n0*Tq!7=bLTY-~V!VrBr4`L_xI z02>34g^QJqi<$i|*nLvI|L0W(e_UVi7ZYq9{&7ixhoQso6$O7|l$HHsR$JS@ayz>y znf{Yng}-n+t9Uw?GAWrl+q*g$|Jpb+vfqRqxP+Zd4PES=RP60-{<4Y+7WOXo&KC9# z#KOvK#MH8e#+G)!b%4tWLf^H>~GHSyR@m3rJJeAZ|4W` zpN!_R{O7s=|G6%|%bNUiU6}u=EEC^9sNVQDr5nWmwseE|Z?11@_sid%emQmoF_RY< z7+KVxsyS9qbzxLg#Mg4#JL{z{I?JgIL>0ag8P`U1x?`~F(?{SHiWwKo^y?X&^47Cg z304J!P{LBvcQV+nY`&$pzlXxKL@;_j%sR|#cR!pQ2VJM6yPE;BSZs@6GHFNJE^7LH$q)r#OofArcP^B0hxe@08E}n2)+kOiFTx*3eO|Bp#lQN*f-7 z`YaykS62_$?1g`_>uw*_$ma7*jNIN<^QHJI?Km3(aIYG{!RD9> zC#8~ylLNV4%y5OgT*lWa3M#tA?JCl?IE4Z?iyHDDu-4y+AL1yP;mWeB(n@o>XtOtU zBn7S+QHaUE@n@~uE~hp`?w^|~HJdt87Jf#0-Hi|$oM$tOZbz$?3S(zS=R)%$ z{@?>qo{exJq#~wiUfrTt;(45XWRBYUl`3&Orp$Bt9Tvqwgf{zo^y%?R&@|JiKl8{$ zC?Q&k)@pTI{5L@k^IrJ~NPC#VAKdAt)fotI7HH&XK1L&RStIJy^G!-^p>qThKR!x4 z%uD3O1FGR-bZ&@?`_-+#b(}PIv>|;L7MtbAf*@@ywq@6mOnd+MAj)6d6n$XRD{-{8 zc-foC^u?xiuYN2e1G4Ek%uIR){?m=Ig+Q|y!t9mwnX;3V&cjckuxF)m(*%vl1q)Zi zfCOr=J{JKP6#`0KE3Ru;0YtO_otHDpMTh&&FUKuIby z8knb8>50_PaeP&$hATj0MJ}3CUaai4bnp$J^ za=46C)b%%XSQ!w=hp>;JxOwYEpRN>+0=23k67e@Bk4)r?@4kKMW4g6gL6^Xx{6y1a zLY^3d;T6As0X5}Q^MJY){Q+kgU zn%uyh{X=PZap606-*B|F3}p+?A3MP#LvJy-^3Yn5so6)3f`WX9l*n*=!sSNAn35`# z(o>!5oWpB7SH${J#?4;fr1^~DeiS;%+Vm(DCl|->B6K8Qd{*k;;mh-9%wt|u!XY5@ z>D)ae=gG=WC6CNkg;41ev=dR`sAl?{5G2c%Cg{O^lwz{R@G1;TkY91JRE}>|-s?qs z4dd&F_0TM6c;hBDmT(fwVJcCy%Aqv#mOVOf>?qm71rE=u3iFt~cyaCr&_ro1&m_CA z<&lxpifkUN&IR$~wJ$NO++P%|tO=?4m+ATcaC{}-JWYa_wrxXp(ad`Xe(jRCCX_u8 zpZxr?L<+M&87Gu5>vemyC%rP@bbrsQZXUdqB`u8}{%O#uc(JVB>)X+jq;G-Jm2b<` zS^IG?N1)?CU+Jsgc^=@dhI1MYJWr@oGI$ZS1Oh@Z|9UUu!^MlH+2!2=w`ww`s0n{w!q?l}>nC5Bt%bcOZ)r@fyQQ!T z``MBqaG%L=k;Tly&k^u~!st5Ux((Oslp#v8H8j6c#2tpc`1u+BOi(d28F(>vOO`eo zO|dUtAsY5sccYD2w_8|N<}hM)hEP%@#MMp?_pTuCf~+Rsh?s=4Qdz|*RNeOKbY{?| zRieU8ax4`ZC;Fq>U6we_0P~tr#OQcDlEW<=zeK@&o%!7@=M(ZqNCY{vSA@cNpurfX zl$P;ntMnM1+dF3JAc>-TuWEr}=n>qX&6_f_YfC;GbLzdGT7<;xE)Xd4EX*ZN^=4tY z(W4(Y*{A*DNND3IVOn&(`5ZTv1GdnW8+DEJio7YV8aJE~q^hMw(UEN&D#R`Tq+(!I zh?vMfgz;1j2X|o!ZO5HD)=!gswKzH*>VTE*;e1h5ScT8vYK4v{(JP3Kw%5(yW)iDp zaU>(scNFzvvv!dv!HavkQmCk&v+I1N%-4ivXeBpMA@-!OOy7#bXX5&ZplX<;5)5(R zoaoS>Q#H@L`iiKVhcp^w6S5dUF1xSgd8f(y?2Q zL*nvav%w5C)UuZk>*ovX%75)TcXXwZHdk+y_?lN9C?r>&0viK8t=Tl&k`r6+I?|qv z?|%65uv5)&x?-N+m9(^GT$7ffzA9TY2g3x+?f!!tF$Us1lMkt z&F>(1FURmgX-QGu!-Lrf{e&i6cMd&(`m(GTyFXly#|jkG(|O_IxHpo*11L`kHM%?5`m zK5FF2pv5zU^!K>e6?4OUgi|uO*d7^zg;od8uKu$6`cKpO*;)9uDD`}#aemxaq4m@Tr$Z?*P*qCoAU5HBGZ-BQuot9Km}6-{O|GCzQI$)sK;(mLn5qlm zGWl!l58sWMBd?4Lkyhr=#4@cO81$^$kLQcveuOL{9y*O#N1q>MJ9bm@t6jS{=EF$F z%m55?XIo9#4(MJI6>k;)qMjrbi-PCN|39~ zLO--VwE9!E`p-2$-c1ute_7Tvsn?8#wvWdAL<`f0HZwR7+BQi{p69|>V=-K}Rg(jA z1XQpb`N0tX*)ud4ow-^_i3vspUHYbdOS z7r*73&a9!~pfob|SePF5c0374HXmHZSgxmXcZ}C|!V?szx%#L811%H$4M@v-!(0xQT4+Koex$OF(T0o7YBicSIaG-rFq{3}^R=_{EaO?F z##~A-55N%dPQ=q9IHPNPm|N^BSN2`QEc4h(rkt&9w7c@K23)Wd_DHf;oENt>)|@^H zXsG04823Aw%G&;R0(>qco7bnix6PKOd}kGE*YA5)XbYbp$+2k8+_JhrFcRrwme@8I*h4%JUK6u(~2ugj4}bj(KT$VJR2 zicp@MPH4z_X&C5;)z?z8NB!Jn7FJfFQdS_%DQ&TjX0F58JJhPK2@#Y)g$uFiB=+r& zDqpbpjwic+Yw4HY-?JH=6vkRltu$D$4QQv0*?xo|(f2$C9cRPD@_c`L>~d=GGj1c>2t5Sp;5Q zpaj4ourWiT%-8pBecFY2Jmd0Wv_a0;8QAdhfrvH`2?}Z*IaaIg{INSx@*$4@^V>tR zWF9$&UzPv4Y1ica(szxm%(#LlJw1NwwB~Zf&b|o@1WX1PyDq}SE8=lni~jusX4ZUx zL-q^J^@K8VzqKthB)#pgB?IEL^Bb2}1*}0%%GqfoP(MY#`EU9Cj6%!Ix1g_=AW1U$ z;mQ;I09S7barn)1O*n)iq(z{lXd5e-6t+wyuQ2ZVOx3wd$CH%V*djJ`f$PRkgkK>| zN`pc*G*$sI^!9$s-R`nJA>{;q!8c96o`^^lD-zyZ>A{;m;KacOA*Xiv=qKTifwm z4uHp!G;rXyzu@)yNX5x`V5@-axCZazN0Jl9^PI)f2}Oj^n$z*KQsIW{$n2Pyw1JIP z?@;8k72rp~(&H;~tJs$gjJAwD2jp{Ul|ooUCKYi)o#TbYJ|e<7L{d=ktFe0+VUSrJ zReb6(Rc8V+sp8iFJKItUp%>Iz7rZ=^=;dqot7q$bU$y$n~*rb)*qG zu0g~%S&+Qz;AS5zPOWES7&LV zYDx@lt<7S%orT!-yfy>NsK51AE+awqw6MqILS*WKgvfV16GEB=tz-Q+%2<66izq#jA$UO8V zltu57HZDcPUeZL>^8<&#(LrxO+QR_)R|4!HzYdb`VX z^u6CL5}Y|Zvv`Co+Y*MrqnJiSzHvmMd$ZUb2e&?y|LKFJS)CHBgtqyk?7k4-rVf&u z->=@fvEDY)E`2wLtI{w#HNpJ&JXbY^BoP$jw~9-BVp@P5k-T2-O=Q;&;J1LLHPH3X z&C)b*CnfipPwGePKIAm&M$67Zi8){TX(G;)DuvI-7)k|=^wVufSD?p%|BXQ!{t=5I zYn&{XNEdh9Ozo~Im%_A(%lql*TTv4a3Q;EGrTwmxzsQGfX-XZp&LDK6>!?mPUHL3; zC%5;)G?p$B(lFur$5^GFe6;QN9lMIq5g59z>D|RD`Jn3aCsuJgAE;+DOMy~NE>sqI zn4rTGfu&lPujUPtT~k5tfs#k(vS^tVix%~MnHLvhvwDni7HXNi*j%gK$CYMd2o~>w-;m*)I`o-2ON9wjgM9Sh z>avPVSI10T8BIxjq;4(>4r~xX;F^M^YCflrUB32U?^dv~4kxproMt8`N@)T8w|W;5_QWXq;dF=1^sGvIH$J3u3(C_Jo7ErOBT+|PYiZ< z66MQ@Z*tYX@-TQE7{@jnkRc_J{nHw)T*J6|w-lz!^Zxmx#?{!XO+JYIK#Y z=!J`v#*;|6HY3iEP<*e*yuHYRsJx#4fQdWpZNj2PvG(P%1#X}M86v+y*_k|0W)KIF zOHyqpH=jFnT9W`5nj}zcK!hB9(7H-05s+th(enhPQSK1RpUq%nM=!&Sh7xtsEf5?8 zd->9nV0EH%xGh0}C4^3o*I1x~yT%u9w(6^ZK1@zlhGFIHSjj(~Tl4nf z&8@eA1)>R)fk=tb)wEWAqyRB8Ftdt3dV_WXgz^m#F?rRa%(aE`PI|kb9x;-U#3ASf zxYEP(ZB9+>dzL5j65yTSu_l6G%VMg_Ob{ou7hKFC;crBjJ~C8)$Hk9^OoPwkAVrfC zS4O=Aed-B;q8L5nB7Ddzp=*NYHGrwI^k)2W?};(J<)mG?k$p=d2$M_&Z)!iMlHAIY zUmdKb=@TL0t56BG_#r;b4d-ri%_xJOvhHVIxE<4M`rhNNX`x0D^5jMlB`9_BgvNa-)d6#RG^k!$ z9(lF0`Q&Gq5T%LfhYNk#YEd03COADjPZm61w@5g0Cfn3`OV!Nt2} zt~9Qf2;K236DU;%S*YZ9ax%>gE!;_d=vE`C&aNJDJLp)H;kxB~td=M-VA^ywv^CL@ z0uTihs|4n}>ZLBpoph&&mM(bKz*Tcp^Ew&7hOA9d3ZT>rm19-F1-IL2KA57+55old zi8Uy}8oT9BTlE~w`4=5xd3Ugrqj!Cr$b;kZS`LvXTHaUSc5j#J(XD%z(30KBex@Pu z;IjPgvp}_B8@EP73q8(HtSas=kSH6U?S8UwUZ37`PsL%5B=3T&q*0foi1m)P_*@U1 z1}sh{%Ppfsc+BjV`l_>8Ns=>9TL>z6$DPzle#W@2*bU-;UBP@J<;Bsj$xo#F(2zg3 zay@sYwG}1t;4@U8$=`J6NQbm_i9`xU6*y$}soAkvcGG)rmFKG&IB|0D1*QM1wJ+?~ zb~N;^BhPB1oYF@K>|NQoBSRI9s8R$*1m)>{oKv9_3%LQaZhbIXORF~ zg^jIY7bf8!YHx7_hax@d-TQdM_FyK-^F=#&waHo)ovYa?I&O0KelcQS~?{cB}@U=)W8S3`)W{kXbJ7#rLU zRG|Ne#tZwmubVbr49^1!Wr$#dd6vWUp^#gqaBu#U^kzDW0SgNB_34-sLnaup(|F-` z{fHD9RSAQw~{-`$ouUV^V_I z`8JmY2$%B*Zoj#m-%qriLU7O@(Qt+^#^qeU6z)CEj~jV zfB)LBi^Ms)`!Jum*%jJ!j!1N@1Nj2+aKr=5oG{-k$RWTG2kH| zH^-E$oBhH5NBegE$OPkn0lZL-92xd~-S^QPFD6Hn&6#sTmEce#({OeXTa-+$u!{iV+z@cjJz@H{8GRmm+8fXsG~^w?z(3V?ey z0&;fNp*UNMAiu1e1~!G1G7KjM)z80@mn&rx8PM#JFpW2>N}>3F?dDR!GK}qiqQERW zjK*``D{JAppJm1T*>8{5HvnM^EGQB1TUvwEryizTo8U8V$Ob9URIY|VkPf=mx8%Um$1@c+`3Gve$&v)(`dw?`Ea%W90ZC#V z&dA=1dLShbRATu2h<>(m+|5-bHf4mcQQ-GN`yAjkzVYS8n>=+1IG(z~yLL4GsSH*_ z0IdpxD6_-)H6{R{%hvpttDMG{L)ItXlyR{#WAr9gS1)KcvC}OB95n+Xsglb^2RS@( zyb88XW(cO78vf6yKeg4THG{Wfn6-_GB{;9uTr*;e^QPuSz|P7Stp$QTW>_JjN-u$} z23tUE398PPEe1_Qg@8x9+aYS;{_B{bqBd}h-{5?eQ{%Y@oSdP`f;_!3Nb0tIZz$pL z(??#AevEN68t?4c{iBe62NKz@9fT4~V-vT4o!LZfU!`zYJv`po+fNEsJ?`UKZ8m_j zuXkiB7OURE*xny{+nJ*NK5mWH@Rs_)#*X@3inL5~i|nu(z$6yt3iAXU5~=mcg z&uIz}hrZ=?wmNP=_cYa9pziJ%Ulx6;Cirmb+^aLnM8|{l}y3oRtp&tlCtU|BO4j-6UA=Fn1pTad2K%D!SAmBA&sPad@Cu z2HP4alJO7T`qXV>ZSFXmTrd;e_N@h$dd`W!{)W|Q&QGAiqh}J7Ha!&EI{gGtTY>%0 zT&Dpky5oci*=9W{jcN;N*}SJ4i%j3WBVT;VxSaAv;{^bXQnbi7Mb#KY2FE4Kp9#P9 zFrA<_hcrOFZGd{^W9p}DlP$~rx(rO&Vhr5H7F0bP;8Sya`O@h1vU1l=xI&>zIoi`~ z^_;K>9WQc?i1y2GD%B1-EW9ckR8+dBu~MKN7E4P~{mgyR_jQ2RQdb|qe(IU?W-`eHtQ%xL<3Rv)?D1sWTR^Zm&|4O~jXm>W{(q}|F( zMktHI9!MK{LG{A6Ithaq_J9F6=c9FXic(J8;SnUQ#NeEf#=v(c;0HWc>4ud;hhEY~ zuzLckKQ`lYWea-?{^BG{>yh5R%DTf2%41eQ1(p4Bh}H z9}gn_bO`3uD?#3U9hd8FV_JMm5kPzAQR`w~YINr#)N%EsgAfZdE$d6ldxBHnMltIu z<9kC)()&(BCli-e5)v9izXfdxx+lRDPJ(yUcFX%25Cb@AwEB^RnSJmJ)ym|x^?TylJ1abRoTY22L26=Syf_T4G z>%-hnCQ($#^(z@qa3sckV#DIdr#+u{yF>@M01maZ^$llMXoWS)rClOw3wb9pKU#Jm zV1(N{Oh18RlDjS9e(LVvrB-b%z|7%7y^lB+T`@@+SQc&S-}cf0_3Wkp>^pC|jMd7( z>ET^}JnAmalO7IWq=li`)D;bE<}*+EnR(C{UZV5STJriI8msi69qmPIrfvJNP}jvo z?^dqZD{jW5%L~1Go;}yf-KUc%H*?j*9uH<=GI3g$3@VyCZ>oqMZ<-jsdf#Hy<;Y!J zDBn_FGB^Qenoh=brze}li!^!0YTjcOh$A0^_;x$In`Oxqp#J<8^jjf_cHn{ZY<0QL zM(EckUq_^o1>_WM=J5`7b64I3e#JB;Ql+Pmk}4qNjD;1B_4i{vNh}92Z$0kzY-VUi z-vVRus+DaR)>SPc9Zmm?z_{ZgNcfebzVR6iDgLRKa@)5DaE%g;h3(Ba-F({dP-ThD z(9Ojd&H8M`PG{&lb;hvYp~i4^74n`uqb^hQcE1l0;$`LsMuagrl9i+;I>Sd@5rN=@ zR*Z=`=+V#(gs=8dFW0hx+|bH&i_BBiLvcb;^}p5C$3o4Cw!+6Zsl3kE5gs)birOv@ zr%9(j&9SR%7#E7YUKi%AwV5=cK z99OLLFm_qRIxx!h)dpVau<7$V69V5YQtk)GwS!WA-Q6x9|UYG z{cpx1|9+10l8+9`#F@Y%PK%9!98$_46M`F@@P0DAk2f0)u{6E_0o#8_OMUKQx40Ac_D%W7`L literal 0 HcmV?d00001 From 2f034085bdad48c192fe5d166092b23f0a7d5e7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Thu, 19 Feb 2026 14:33:10 +0100 Subject: [PATCH 06/41] UFAL/Nw version metadata issues (#1236) * Issue ufal/clarin-dspace#1266: dc.date.available and dc.relation.replaces metadata not cleared properly (ufal/clarin-dspace#1307) * Issue ufal/clarin-dspace#1266: dc.date.available and dc.relation.replaces metadata not cleaned properly in new item version * resolve MR comments - update ignoredMetadataFields in versioning-service.xml * update ClarinVersionedHandleIdentifierProviderIT test to check dc.identifier.uri metadata for new version (cherry picked from commit 7ffaf9a807d94da0cec77b67a9cad2d1fc7c5f38) * Issue 1319: do not copy dc.identifier.doi metadata when new item version is created (cherry picked from commit 1b7ed17228c5f1260b250361fa87f49d9af0e14a) --------- Co-authored-by: Milan Kuchtiak --- .../DefaultItemVersionProvider.java | 2 +- ...inVersionedHandleIdentifierProviderIT.java | 155 ++++++++++++++++++ .../config/spring/api/versioning-service.xml | 5 +- 3 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 dspace-api/src/test/java/org/dspace/identifier/ClarinVersionedHandleIdentifierProviderIT.java diff --git a/dspace-api/src/main/java/org/dspace/versioning/DefaultItemVersionProvider.java b/dspace-api/src/main/java/org/dspace/versioning/DefaultItemVersionProvider.java index 5a2695b9a61e..07d57496cfe7 100644 --- a/dspace-api/src/main/java/org/dspace/versioning/DefaultItemVersionProvider.java +++ b/dspace-api/src/main/java/org/dspace/versioning/DefaultItemVersionProvider.java @@ -202,7 +202,7 @@ protected void copyRelationships( */ private void manageRelationMetadata(Context c, Item itemNew, Item previousItem) throws SQLException { // Remove copied `dc.relation.replaces` metadata for the new item. - itemService.clearMetadata(c, itemNew, "dc", "relation", "replaces", null); + itemService.clearMetadata(c, itemNew, "dc", "relation", "replaces", Item.ANY); // Add metadata `dc.relation.replaces` to the new item. // The metadata value is: `dc.identifier.uri` from the previous item. diff --git a/dspace-api/src/test/java/org/dspace/identifier/ClarinVersionedHandleIdentifierProviderIT.java b/dspace-api/src/test/java/org/dspace/identifier/ClarinVersionedHandleIdentifierProviderIT.java new file mode 100644 index 000000000000..355ed2a8fb90 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/identifier/ClarinVersionedHandleIdentifierProviderIT.java @@ -0,0 +1,155 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.identifier; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.startsWith; +import static org.hamcrest.MatcherAssert.assertThat; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; +import java.util.TimeZone; + +import org.dspace.AbstractIntegrationTestWithDatabase; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.builder.VersionBuilder; +import org.dspace.content.Collection; +import org.dspace.content.Item; +import org.dspace.content.MetadataValue; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.InstallItemService; +import org.dspace.content.service.ItemService; +import org.dspace.kernel.ServiceManager; +import org.dspace.services.factory.DSpaceServicesFactory; +import org.dspace.workflow.WorkflowItem; +import org.dspace.workflow.WorkflowItemService; +import org.dspace.workflow.factory.WorkflowServiceFactory; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit Tests for ClarinVersionedHandleIdentifierProvider + * + * @authorMilan Kuchtiak + */ +public class ClarinVersionedHandleIdentifierProviderIT extends AbstractIntegrationTestWithDatabase { + private IdentifierServiceImpl identifierService; + private InstallItemService installItemService; + private ItemService itemService; + private WorkflowItemService workflowItemService; + + private Collection collection; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + context.turnOffAuthorisationSystem(); + + ServiceManager serviceManager = DSpaceServicesFactory.getInstance().getServiceManager(); + identifierService = serviceManager.getServicesByType(IdentifierServiceImpl.class).get(0); + + itemService = ContentServiceFactory.getInstance().getItemService(); + installItemService = ContentServiceFactory.getInstance().getInstallItemService(); + workflowItemService = WorkflowServiceFactory.getInstance().getWorkflowItemService(); + + // Clean out providers to avoid any being used for creation of community and collection + identifierService.setProviders(new ArrayList<>()); + + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + collection = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + } + + @Test + public void testNewVersionMetadata() throws Exception { + registerProvider(ClarinVersionedHandleIdentifierProvider.class); + Item itemV1 = ItemBuilder.createItem(context, collection) + .withTitle("First version") + .build(); + + // new item "dc.relation.replaces" metadata has to be set to this value + String itemV1HandleRef = itemService.getMetadataFirstValue(itemV1, "dc", "identifier", "uri", Item.ANY); + + // set "dc.relation.replaces" metadata on itemV1 + itemService.addMetadata(context, itemV1, "dc", "relation", "replaces", null, "some_value"); + // replace "dc.date.available" metadata on itemV1 to some old value + itemService.clearMetadata(context, itemV1, "dc", "date", "available", Item.ANY); + itemService.addMetadata(context, itemV1, "dc", "date", "available", null, "2020-01-01"); + // simulate itemV1 having a DOI identifier assigned + itemService.addMetadata(context, itemV1, "dc", "identifier", "doi", null, + "https://handle.stage.datacite.org/10.5072/dspace-1"); + + Item itemV2 = VersionBuilder.createVersion(context, itemV1, "Second version").build().getItem(); + + // check that "dc.date.available", metadata is not copied to itemV2 + assertThat(itemService.getMetadata(itemV2, "dc", "date", "available", Item.ANY).size(), equalTo(0)); + + // check that "dc.identifier.uri", metadata is not copied to itemV2 + assertThat(itemService.getMetadata(itemV2, "dc", "identifier", "uri", Item.ANY).size(), equalTo(0)); + + // check that "dc.identifier.doi", metadata is not copied to itemV2 + assertThat(itemService.getMetadata(itemV2, "dc", "identifier", "doi", Item.ANY).size(), equalTo(0)); + + // check that "dc.relation.replaces" points to itemV1 + List metadataValues = itemService.getMetadata(itemV2, "dc", "relation", "replaces", Item.ANY); + assertThat(metadataValues.size(), equalTo(1)); + assertThat(metadataValues.get(0).getValue(), equalTo(itemV1HandleRef)); + + WorkflowItem workflowItem = workflowItemService.create(context, itemV2, collection); + Item installedItem = installItemService.installItem(context, workflowItem); + + // get current date + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(System.currentTimeMillis()); + calendar.setTimeZone(TimeZone.getTimeZone("UTC")); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + String date = sdf.format(calendar.getTime()); + + // check that "dc.relation.replaces" points to itemV1 + metadataValues = itemService.getMetadata(installedItem, "dc", "relation", "replaces", Item.ANY); + assertThat(metadataValues.size(), equalTo(1)); + assertThat(metadataValues.get(0).getValue(), equalTo(itemV1HandleRef)); + + // Check that itemV2 has the correct "dc.date.available" metadata set to current date + metadataValues = itemService.getMetadata(installedItem, "dc", "date", "available", Item.ANY); + assertThat(metadataValues.size(), equalTo(1)); + assertThat(metadataValues.get(0).getValue(), startsWith(date)); + + // check "dc.identifier.uri" metadata has new value different from itemV1 + metadataValues = itemService.getMetadata(installedItem, "dc", "identifier", "uri", Item.ANY); + assertThat(metadataValues.size(), equalTo(1)); + assertThat(metadataValues.get(0).getValue(), not(itemV1HandleRef)); + } + + private void registerProvider(Class type) { + // Register our new provider + IdentifierProvider identifierProvider = + (IdentifierProvider) DSpaceServicesFactory.getInstance().getServiceManager() + .getServiceByName(type.getName(), type); + if (identifierProvider == null) { + DSpaceServicesFactory.getInstance().getServiceManager().registerServiceClass(type.getName(), type); + identifierProvider = (IdentifierProvider) DSpaceServicesFactory.getInstance().getServiceManager() + .getServiceByName(type.getName(), type); + } + + // Overwrite the identifier-service's providers with the new one to ensure only this provider is used + identifierService = DSpaceServicesFactory.getInstance().getServiceManager() + .getServicesByType(IdentifierServiceImpl.class).get(0); + identifierService.setProviders(new ArrayList<>()); + identifierService.setProviders(List.of(identifierProvider)); + } +} diff --git a/dspace/config/spring/api/versioning-service.xml b/dspace/config/spring/api/versioning-service.xml index 1a5358edd777..d8b76299e432 100644 --- a/dspace/config/spring/api/versioning-service.xml +++ b/dspace/config/spring/api/versioning-service.xml @@ -21,11 +21,14 @@ dc.date.accessioned + dc.date.available dc.description.provenance + dc.identifier.doi dc.identifier.uri + dc.relation.replaces - + From 0f1ff8b57e772013778b6521c65240bba39816e7 Mon Sep 17 00:00:00 2001 From: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> Date: Thu, 26 Feb 2026 08:29:04 +0100 Subject: [PATCH 07/41] UFAL/Fix: add bitstream download-by-handle endpoint for curl instructions (#1252) * fix: add bitstream download-by-handle endpoint for curl instructions Adds GET /api/core/bitstreams/handle/{prefix}/{suffix}/{filename} endpoint that directly serves bitstream content by item handle and filename. This resolves the issue where curl download instructions generated by the UI produced URLs pointing to non-existent backend endpoints, resulting in 404 errors when users attempted to download files via command line. The new endpoint resolves the handle to an Item, finds the bitstream by exact filename in ORIGINAL bundles, and streams the raw content with correct Content-Type and Content-Disposition headers. Refs: dataquest-dev/dspace-angular#1210 * Fixed compliing errors * Small refactoring - use constants and removed unnecessary changes * added comments, return 404 status instead of 402 * unauthorized instead of forbidden * fix: use RFC 5987 Content-Disposition for non-ASCII filenames curl -J on Windows cannot create files with non-ASCII characters (e.g. diacritics like e/a) from a raw UTF-8 Content-Disposition filename header. Uses filename*=UTF-8''percent-encoded-name (RFC 5987/6266) which curl properly decodes. Also includes an ASCII fallback in filename param. * fix: move context.complete() after streaming to prevent truncated downloads context.complete() was called before bitstreamService.retrieve(), closing the DB connection and causing 'end of response with X bytes missing' errors. Now context.complete() is called only after the full content has been streamed. For S3 redirect and HEAD paths, context.complete() remains before return since no streaming is needed. * fix: use real UTF-8 filename in Content-Disposition instead of ASCII fallback The filename parameter now contains the original name (with diacritics like e/a) instead of replacing non-ASCII chars with underscores. Characters in the ISO-8859-1 range are transmitted correctly by Tomcat and understood by curl on Western/Central-European systems. The filename* parameter still provides RFC 5987 percent-encoded UTF-8 for modern clients (curl 7.56+). * fix: revert to ASCII fallback in Content-Disposition, add edge-case tests Content-Disposition filename parameter now uses ASCII fallback (non-ASCII replaced with underscore) per RFC 6266. Modern clients use filename* (RFC 5987) which has the full UTF-8 name. The curl command no longer relies on Content-Disposition at all (uses -o instead of -OJ). New integration tests for edge cases: - Multiple dots in filename (archive.v2.1.tar.gz) - Double quotes in filename (escaped in Content-Disposition) - CJK characters (beyond ISO-8859-1) - Same filename in ORIGINAL and TEXT bundles (only ORIGINAL served) * fix: resolve compilation errors and fix IT test assertions - Remove duplicate HttpStatus import (apache vs spring) - Add missing MediaType import (spring) - Fix Content-Type assertion to include charset=UTF-8 - Use URI.create() for pre-encoded URLs in tests to prevent double-encoding (%25) rejection by StrictHttpFirewall All 15 integration tests pass. * test: add complex filename test (diacritics, plus, hash, unmatched paren) New IT test for filename 'Media (+)#9) ano' verifying correct URL decoding, Content-Disposition encoding, and content delivery. 16/16 tests pass. * fix authorization, comments, tests * fix: change expected status from 401 to 403 for authenticated non-admin user The test downloadBitstreamByHandleUnauthorizedForNonAdmin uses getClient(token) which means the user IS authenticated. The controller correctly returns 403 (Forbidden) for authenticated users without access, not 401 (Unauthorized). 401 is only for anonymous/unauthenticated requests. --------- Co-authored-by: Paurikova2 --- .../rest/BitstreamByHandleRestController.java | 322 ++++++++++ ...ionCCLicenseUrlResourceHalLinkFactory.java | 1 + .../BitstreamByHandleRestControllerIT.java | 599 ++++++++++++++++++ 3 files changed, 922 insertions(+) create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/BitstreamByHandleRestController.java create mode 100644 dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamByHandleRestControllerIT.java diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/BitstreamByHandleRestController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/BitstreamByHandleRestController.java new file mode 100644 index 000000000000..36cdffc9b201 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/BitstreamByHandleRestController.java @@ -0,0 +1,322 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest; + +import static org.dspace.core.Constants.CONTENT_BUNDLE_NAME; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; +import java.util.List; +import java.util.Objects; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.InternalServerErrorException; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.dspace.app.rest.model.BitstreamRest; +import org.dspace.app.rest.utils.ContextUtil; +import org.dspace.app.statistics.clarin.ClarinMatomoBitstreamTracker; +import org.dspace.authorize.AuthorizeException; +import org.dspace.authorize.service.AuthorizeService; +import org.dspace.content.Bitstream; +import org.dspace.content.BitstreamFormat; +import org.dspace.content.Bundle; +import org.dspace.content.DSpaceObject; +import org.dspace.content.Item; +import org.dspace.content.service.BitstreamService; +import org.dspace.core.Constants; +import org.dspace.core.Context; +import org.dspace.handle.service.HandleService; +import org.dspace.services.ConfigurationService; +import org.dspace.services.EventService; +import org.dspace.storage.bitstore.S3BitStoreService; +import org.dspace.storage.bitstore.service.S3DirectDownloadService; +import org.dspace.usage.UsageEvent; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +/** + * This controller provides a direct download endpoint for bitstreams + * identified by an Item handle and the bitstream filename. + * + *

Endpoint: {@code GET /api/core/bitstreams/handle/{prefix}/{suffix}/{filename}}

+ * + *

This is used by the command-line download instructions (curl commands) + * shown on the item page in the UI. Only bitstreams in ORIGINAL bundles are served.

+ * + *

Note: {@code @PreAuthorize} is not used because authorization depends on the resolved + * bitstream (looked up by handle + filename), not on a UUID path variable. Authorization + * is explicitly checked via {@link AuthorizeService#authorizeAction} after the bitstream + * is resolved.

+ */ +@RestController +@RequestMapping("/api/" + BitstreamRest.CATEGORY + "/" + BitstreamRest.PLURAL_NAME + "/handle") +public class BitstreamByHandleRestController { + + private static final Logger log = + org.apache.logging.log4j.LogManager.getLogger(BitstreamByHandleRestController.class); + + private static final int BUFFER_SIZE = 4096 * 10; + + @Autowired + private BitstreamService bitstreamService; + + @Autowired + private HandleService handleService; + + @Autowired + private AuthorizeService authorizeService; + + @Autowired + private EventService eventService; + + @Autowired + private ConfigurationService configurationService; + + @Autowired + private ClarinMatomoBitstreamTracker matomoBitstreamTracker; + + @Autowired + private S3DirectDownloadService s3DirectDownloadService; + + @Autowired + private S3BitStoreService s3BitStoreService; + + /** + * Download a bitstream by item handle and filename. + * + * @param prefix the handle prefix (e.g. "11234") + * @param suffix the handle suffix (e.g. "1-5814") + * @param filename the bitstream filename (e.g. "pdtvallex-4.5.xml") + * @param request the HTTP request + * @param response the HTTP response + * @throws IOException if an I/O error occurs during streaming + */ + @RequestMapping(method = {RequestMethod.GET, RequestMethod.HEAD}, + value = "/{prefix}/{suffix}/{filename:.+}") + public void downloadBitstreamByHandle(@PathVariable String prefix, + @PathVariable String suffix, + @PathVariable String filename, + HttpServletRequest request, + HttpServletResponse response) throws IOException { + String handle = prefix + "/" + suffix; + + Context context = ContextUtil.obtainContext(request); + if (Objects.isNull(context)) { + log.error("Cannot obtain the context from the request."); + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, + "Cannot obtain the context from the request."); + return; + } + + try { + // Resolve handle to DSpaceObject + DSpaceObject dso = handleService.resolveToObject(context, handle); + if (Objects.isNull(dso) || !(dso instanceof Item)) { + log.warn("Handle '{}' does not resolve to a valid Item.", handle); + response.sendError(HttpServletResponse.SC_NOT_FOUND, + "Handle '" + handle + "' does not resolve to a valid item."); + return; + } + + Item item = (Item) dso; + Bitstream bitstream = findBitstreamByName(item, filename); + + if (bitstream == null) { + log.warn("No bitstream with name '{}' found for handle '{}'.", filename, handle); + response.sendError(HttpServletResponse.SC_NOT_FOUND, + "Bitstream '" + filename + "' not found for handle '" + handle + "'."); + return; + } + + // Authorization is checked explicitly here (not via @PreAuthorize) because the + // bitstream identity is resolved from handle+filename, not from a UUID path variable. + authorizeService.authorizeAction(context, bitstream, Constants.READ); + + // Fire usage event for download statistics + if (StringUtils.isBlank(request.getHeader("Range"))) { + eventService.fireEvent( + new UsageEvent( + UsageEvent.Action.VIEW, + request, + context, + bitstream)); + } + + // Retrieve content metadata + BitstreamFormat format = bitstream.getFormat(context); + String mimeType = (format != null) ? format.getMIMEType() : MediaType.APPLICATION_OCTET_STREAM_VALUE; + String name = StringUtils.isNotBlank(bitstream.getName()) + ? bitstream.getName() : bitstream.getID().toString(); + + response.setContentType(mimeType); + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, + buildContentDisposition(name)); + long size = bitstream.getSizeBytes(); + if (size > 0) { + response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(size)); + } + + // Track download in Matomo + matomoBitstreamTracker.trackBitstreamDownload(context, request, bitstream, false); + + // Check for S3 direct download support + boolean s3DirectDownload = configurationService + .getBooleanProperty("s3.download.direct.enabled"); + boolean s3AssetstoreEnabled = configurationService + .getBooleanProperty("assetstore.s3.enabled"); + if (s3DirectDownload && s3AssetstoreEnabled) { + boolean hasOriginalBundle = bitstream.getBundles().stream() + .anyMatch(bundle -> CONTENT_BUNDLE_NAME.equals(bundle.getName())); + if (hasOriginalBundle) { + // Close the DB connection before redirecting + context.complete(); + redirectToS3DownloadUrl(name, bitstream.getInternalId(), response); + return; + } + } + + if (RequestMethod.HEAD.name().equals(request.getMethod())) { + // HEAD request — only headers, no body + context.complete(); + return; + } + + // Stream the bitstream content. The context must remain open because + // bitstreamService.retrieve() needs an active DB connection / assetstore session. + Context downloadContext = null; + boolean downloadContextCompleted = false; + try { + downloadContext = new Context(); + try (InputStream is = bitstreamService.retrieve(downloadContext, bitstream)) { + byte[] buffer = new byte[BUFFER_SIZE]; + int bytesRead; + while ((bytesRead = is.read(buffer)) != -1) { + response.getOutputStream().write(buffer, 0, bytesRead); + } + response.getOutputStream().flush(); + } + downloadContext.complete(); + downloadContextCompleted = true; + } finally { + if (downloadContext != null && !downloadContextCompleted) { + downloadContext.abort(); + } + } + // Close DB connection after streaming is complete + context.complete(); + } catch (AuthorizeException e) { + log.warn("Unauthorized access to bitstream '{}' for handle '{}'.", filename, handle); + if (context.getCurrentUser() == null) { + response.sendError(HttpServletResponse.SC_UNAUTHORIZED, + "You are not authorized to download this file."); + } else { + response.sendError(HttpServletResponse.SC_FORBIDDEN, + "You are not authorized to download this file."); + } + } catch (SQLException e) { + log.error("Database error while downloading bitstream '{}' for handle '{}': {}", + filename, handle, e.getMessage()); + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, + "An internal error occurred."); + } + } + + /** + * Redirect to an S3 presigned URL for direct download. + * + * @param bitName the bitstream filename + * @param bitInternalId the internal storage ID + * @param response the HTTP response to send the redirect on + */ + private void redirectToS3DownloadUrl(String bitName, String bitInternalId, + HttpServletResponse response) throws IOException { + try { + String bucket = configurationService.getProperty("assetstore.s3.bucketName", ""); + if (StringUtils.isBlank(bucket)) { + throw new InternalServerErrorException("S3 bucket name is not configured"); + } + + String bitstreamPath = s3BitStoreService.getFullKey(bitInternalId); + if (StringUtils.isBlank(bitstreamPath)) { + throw new InternalServerErrorException( + "Failed to get bitstream path for internal ID: " + bitInternalId); + } + + int expirationTime = configurationService + .getIntProperty("s3.download.direct.expiration", 3600); + String presignedUrl = s3DirectDownloadService + .generatePresignedUrl(bucket, bitstreamPath, expirationTime, bitName); + + if (StringUtils.isBlank(presignedUrl)) { + throw new InternalServerErrorException( + "Failed to generate presigned URL for bitstream: " + bitInternalId); + } + + response.setStatus(HttpStatus.FOUND.value()); + response.setHeader(HttpHeaders.LOCATION, URI.create(presignedUrl).toString()); + } catch (Exception e) { + log.error("Error generating S3 presigned URL for bitstream: {}", bitInternalId, e); + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, + "Error generating download URL."); + } + } + + /** + * Build a Content-Disposition header value using RFC 5987 encoding. + * Includes both {@code filename} (ASCII fallback) and {@code filename*} + * (UTF-8 percent-encoded) so that curl -J and browsers can save files + * with non-ASCII characters in the name correctly. + * + * @param name the original filename + * @return the Content-Disposition header value + */ + private String buildContentDisposition(String name) { + // RFC 5987 percent-encoding for filename* + String encoded = URLEncoder.encode(name, StandardCharsets.UTF_8) + .replace("+", "%20"); + // ASCII fallback: replace non-ASCII chars with underscore, escape quotes. + // Modern clients use filename* (RFC 5987 / RFC 6266) with real UTF-8 name. + String asciiFallback = name.replaceAll("[^\\x20-\\x7E]", "_") + .replace("\\", "\\\\") + .replace("\"", "\\\""); + return String.format("attachment; filename=\"%s\"; filename*=UTF-8''%s", + asciiFallback, encoded); + } + + /** + * Find a bitstream by name in the ORIGINAL bundles of an item. + * Bitstreams in other bundles (THUMBNAIL, TEXT, LICENSE, etc.) are not returned. + * + * @param item the item to search + * @param filename the exact filename to match + * @return the matching Bitstream, or null if not found + */ + private Bitstream findBitstreamByName(Item item, String filename) { + List bundles = item.getBundles(CONTENT_BUNDLE_NAME); + for (Bundle bundle : bundles) { + for (Bitstream bitstream : bundle.getBitstreams()) { + if (StringUtils.equals(bitstream.getName(), filename)) { + return bitstream; + } + } + } + return null; + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/link/process/SubmissionCCLicenseUrlResourceHalLinkFactory.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/link/process/SubmissionCCLicenseUrlResourceHalLinkFactory.java index 07d5e46c61e0..6328f1c56f3c 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/link/process/SubmissionCCLicenseUrlResourceHalLinkFactory.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/link/process/SubmissionCCLicenseUrlResourceHalLinkFactory.java @@ -54,6 +54,7 @@ protected void addLinks(SubmissionCCLicenseUrlResource halResource, final Pageab SubmissionCCLicenseUrlRest.CATEGORY, SubmissionCCLicenseUrlRest.PLURAL, "rightsByQuestions", null, null, null, null, new LinkedMultiValueMap<>())); for (String key : parameterMap.keySet()) { + // Add all current request parameters to the URI being built. uriComponentsBuilder.queryParam(key, parameterMap.get(key)); } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamByHandleRestControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamByHandleRestControllerIT.java new file mode 100644 index 000000000000..52909c42b6ec --- /dev/null +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/BitstreamByHandleRestControllerIT.java @@ -0,0 +1,599 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest; + +import static org.hamcrest.Matchers.equalTo; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.head; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.io.InputStream; +import java.net.URI; + +import org.apache.commons.codec.CharEncoding; +import org.apache.commons.io.IOUtils; +import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.authorize.service.AuthorizeService; +import org.dspace.builder.BitstreamBuilder; +import org.dspace.builder.BundleBuilder; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.builder.ResourcePolicyBuilder; +import org.dspace.content.Bitstream; +import org.dspace.content.Bundle; +import org.dspace.content.Collection; +import org.dspace.content.Item; +import org.dspace.content.service.BitstreamService; +import org.dspace.core.Constants; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpHeaders; + +/** + * Integration tests for {@link BitstreamByHandleRestController}. + */ +public class BitstreamByHandleRestControllerIT extends AbstractControllerIntegrationTest { + + private static final String ENDPOINT_BASE = "/api/core/bitstreams/handle"; + + @Autowired + AuthorizeService authorizeService; + + @Autowired + BitstreamService bitstreamService; + + @Test + public void downloadBitstreamByHandle() throws Exception { + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + Item item = ItemBuilder.createItem(context, col) + .withAuthor("Test Author") + .build(); + String bitstreamContent = "TestBitstreamContent"; + Bitstream bitstream; + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + bitstream = BitstreamBuilder.createBitstream(context, item, is) + .withName("testfile.txt") + .withDescription("A test file") + .withMimeType("text/plain") + .build(); + } + context.restoreAuthSystemState(); + + String handle = item.getHandle(); + String[] handleParts = handle.split("/"); + + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + "/testfile.txt")) + .andExpect(status().isOk()) + .andExpect(header().string(HttpHeaders.CONTENT_DISPOSITION, + equalTo("attachment; filename=\"testfile.txt\"; filename*=UTF-8''testfile.txt"))) + .andExpect(header().string(HttpHeaders.CONTENT_TYPE, "text/plain;charset=UTF-8")) + .andExpect(content().string(bitstreamContent)); + } + + @Test + public void downloadBitstreamByHandleMultipleFiles() throws Exception { + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + Item item = ItemBuilder.createItem(context, col) + .withAuthor("Test Author") + .build(); + + String content1 = "FileOneContent"; + String content2 = "FileTwoContent"; + try (InputStream is1 = IOUtils.toInputStream(content1, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, item, is1) + .withName("file1.txt") + .withMimeType("text/plain") + .build(); + } + try (InputStream is2 = IOUtils.toInputStream(content2, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, item, is2) + .withName("file2.txt") + .withMimeType("text/plain") + .build(); + } + context.restoreAuthSystemState(); + + String handle = item.getHandle(); + String[] handleParts = handle.split("/"); + + // Download first file + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + "/file1.txt")) + .andExpect(status().isOk()) + .andExpect(content().string(content1)); + + // Download second file + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + "/file2.txt")) + .andExpect(status().isOk()) + .andExpect(content().string(content2)); + } + + @Test + public void downloadBitstreamByHandleUnauthorizedForNonAdmin() throws Exception { + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + Item item = ItemBuilder.createItem(context, col) + .withAuthor("Test Author") + .build(); + String bitstreamContent = "RestrictedContent"; + Bitstream bitstream; + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + bitstream = BitstreamBuilder.createBitstream(context, item, is) + .withName("restricted.txt") + .withMimeType("text/plain") + .build(); + } + // Remove all read policies from the bitstream + authorizeService.removeAllPolicies(context, bitstream); + // Add a read policy only for admin + ResourcePolicyBuilder.createResourcePolicy(context, admin, null) + .withDspaceObject(bitstream) + .withAction(Constants.READ) + .build(); + context.restoreAuthSystemState(); + String handle = item.getHandle(); + String[] handleParts = handle.split("/"); + // Authenticated non-admin user should get 403 (Forbidden) + String token = getAuthToken(eperson.getEmail(), password); + getClient(token).perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + "/restricted.txt")) + .andExpect(status().isForbidden()); + } + + @Test + public void downloadBitstreamByHandleInvalidHandle() throws Exception { + getClient().perform(get(ENDPOINT_BASE + "/99999/99999/nonexistent.txt")) + .andExpect(status().isNotFound()); + } + + @Test + public void downloadBitstreamByHandleMissingFile() throws Exception { + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + Item item = ItemBuilder.createItem(context, col) + .withAuthor("Test Author") + .build(); + String bitstreamContent = "SomeContent"; + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, item, is) + .withName("existing.txt") + .withMimeType("text/plain") + .build(); + } + context.restoreAuthSystemState(); + + String handle = item.getHandle(); + String[] handleParts = handle.split("/"); + + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + "/nonexistent.txt")) + .andExpect(status().isNotFound()); + } + + @Test + public void downloadBitstreamByHandleSpecialCharInFilename() throws Exception { + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + Item item = ItemBuilder.createItem(context, col) + .withAuthor("Test Author") + .build(); + String bitstreamContent = "SpecialCharContent"; + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, item, is) + .withName("my file (2).txt") + .withMimeType("text/plain") + .build(); + } + context.restoreAuthSystemState(); + + String handle = item.getHandle(); + String[] handleParts = handle.split("/"); + + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + "/my file (2).txt")) + .andExpect(status().isOk()) + .andExpect(header().string(HttpHeaders.CONTENT_DISPOSITION, + equalTo("attachment; filename=\"my file (2).txt\"; " + + "filename*=UTF-8''my%20file%20%282%29.txt"))) + .andExpect(content().string(bitstreamContent)); + } + + @Test + public void downloadBitstreamByHandleUtf8Filename() throws Exception { + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + Item item = ItemBuilder.createItem(context, col) + .withAuthor("Test Author") + .build(); + // Filename with diacritics: "Médiá (3).jfif" + String utf8Name = "M\u00e9di\u00e1 (3).jfif"; + String bitstreamContent = "Utf8FilenameContent"; + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, item, is) + .withName(utf8Name) + .withMimeType("image/jpeg") + .build(); + } + context.restoreAuthSystemState(); + + String handle = item.getHandle(); + String[] handleParts = handle.split("/"); + + // Use URI.create to pass a pre-encoded URL — get(String) would double-encode %C3 to %25C3 + getClient().perform(get(URI.create(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + + "/M%C3%A9di%C3%A1%20(3).jfif"))) + .andExpect(status().isOk()) + .andExpect(header().string(HttpHeaders.CONTENT_DISPOSITION, + // ASCII fallback replaces non-ASCII with underscore; filename* has UTF-8 encoding + equalTo("attachment; filename=\"M_di_ (3).jfif\"; " + + "filename*=UTF-8''M%C3%A9di%C3%A1%20%283%29.jfif"))) + .andExpect(content().string(bitstreamContent)); + } + + @Test + public void downloadBitstreamByHandleUnauthorized() throws Exception { + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + Item item = ItemBuilder.createItem(context, col) + .withAuthor("Test Author") + .build(); + + String bitstreamContent = "RestrictedContent"; + Bitstream bitstream; + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + bitstream = BitstreamBuilder.createBitstream(context, item, is) + .withName("restricted.txt") + .withMimeType("text/plain") + .build(); + } + + // Remove all read policies from the bitstream + authorizeService.removeAllPolicies(context, bitstream); + // Add a read policy only for admin + ResourcePolicyBuilder.createResourcePolicy(context, admin, null) + .withDspaceObject(bitstream) + .withAction(Constants.READ) + .build(); + + context.restoreAuthSystemState(); + + String handle = item.getHandle(); + String[] handleParts = handle.split("/"); + + // Anonymous user should get 401 + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + "/restricted.txt")) + .andExpect(status().isUnauthorized()); + } + + @Test + public void headRequestBitstreamByHandle() throws Exception { + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + Item item = ItemBuilder.createItem(context, col) + .withAuthor("Test Author") + .build(); + String bitstreamContent = "HeadRequestContent"; + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, item, is) + .withName("headtest.txt") + .withMimeType("text/plain") + .build(); + } + context.restoreAuthSystemState(); + + String handle = item.getHandle(); + String[] handleParts = handle.split("/"); + + getClient().perform(head(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + "/headtest.txt")) + .andExpect(status().isOk()) + .andExpect(header().string(HttpHeaders.CONTENT_DISPOSITION, + equalTo("attachment; filename=\"headtest.txt\"; filename*=UTF-8''headtest.txt"))); + } + + @Test + public void downloadBitstreamByHandleForbidden() throws Exception { + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + Item item = ItemBuilder.createItem(context, col) + .withAuthor("Test Author") + .build(); + + String bitstreamContent = "ForbiddenContent"; + Bitstream bitstream; + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + bitstream = BitstreamBuilder.createBitstream(context, item, is) + .withName("admin-only.txt") + .withMimeType("text/plain") + .build(); + } + + // Remove all read policies and grant access only to admin + authorizeService.removeAllPolicies(context, bitstream); + ResourcePolicyBuilder.createResourcePolicy(context, admin, null) + .withDspaceObject(bitstream) + .withAction(Constants.READ) + .build(); + + context.restoreAuthSystemState(); + + String handle = item.getHandle(); + String[] handleParts = handle.split("/"); + + // Authenticated non-admin user should get 403 (Forbidden) + String token = getAuthToken(eperson.getEmail(), password); + getClient(token).perform( + get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + "/admin-only.txt")) + .andExpect(status().isForbidden()); + } + + @Test + public void downloadBitstreamFromNonOriginalBundle() throws Exception { + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + Item item = ItemBuilder.createItem(context, col) + .withAuthor("Test Author") + .build(); + + // Place a bitstream only in the TEXT bundle (not ORIGINAL) + Bundle textBundle = BundleBuilder.createBundle(context, item) + .withName("TEXT") + .build(); + String bitstreamContent = "ExtractedTextContent"; + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, textBundle, is) + .withName("extracted.txt") + .withMimeType("text/plain") + .build(); + } + context.restoreAuthSystemState(); + + String handle = item.getHandle(); + String[] handleParts = handle.split("/"); + + // Bitstream in TEXT bundle should not be found by this endpoint + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + "/extracted.txt")) + .andExpect(status().isNotFound()); + } + + @Test + public void downloadBitstreamByHandleMultipleDots() throws Exception { + // Verify that Spring {filename:.+} correctly captures filenames with multiple dots + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + Item item = ItemBuilder.createItem(context, col) + .withAuthor("Test Author") + .build(); + String bitstreamContent = "TarGzContent"; + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, item, is) + .withName("archive.v2.1.tar.gz") + .withMimeType("application/gzip") + .build(); + } + context.restoreAuthSystemState(); + + String handle = item.getHandle(); + String[] handleParts = handle.split("/"); + + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + + "/archive.v2.1.tar.gz")) + .andExpect(status().isOk()) + .andExpect(header().string(HttpHeaders.CONTENT_DISPOSITION, + equalTo("attachment; filename=\"archive.v2.1.tar.gz\"; " + + "filename*=UTF-8''archive.v2.1.tar.gz"))) + .andExpect(content().string(bitstreamContent)); + } + + @Test + public void downloadBitstreamByHandleQuoteInFilename() throws Exception { + // Verify double quotes in filename are escaped in Content-Disposition + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + Item item = ItemBuilder.createItem(context, col) + .withAuthor("Test Author") + .build(); + String bitstreamContent = "QuoteContent"; + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, item, is) + .withName("file \"quoted\".txt") + .withMimeType("text/plain") + .build(); + } + context.restoreAuthSystemState(); + + String handle = item.getHandle(); + String[] handleParts = handle.split("/"); + + // Use URI.create to pass a pre-encoded URL — get(String) would double-encode %22 to %2522 + getClient().perform(get(URI.create(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + + "/file%20%22quoted%22.txt"))) + .andExpect(status().isOk()) + .andExpect(header().string(HttpHeaders.CONTENT_DISPOSITION, + equalTo("attachment; filename=\"file \\\"quoted\\\".txt\"; " + + "filename*=UTF-8''file%20%22quoted%22.txt"))) + .andExpect(content().string(bitstreamContent)); + } + + @Test + public void downloadBitstreamByHandleCjkFilename() throws Exception { + // Verify CJK characters (beyond ISO-8859-1) are handled correctly + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + Item item = ItemBuilder.createItem(context, col) + .withAuthor("Test Author") + .build(); + // "日本語.txt" — three CJK characters + String cjkName = "\u65e5\u672c\u8a9e.txt"; + String bitstreamContent = "CjkContent"; + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, item, is) + .withName(cjkName) + .withMimeType("text/plain") + .build(); + } + context.restoreAuthSystemState(); + + String handle = item.getHandle(); + String[] handleParts = handle.split("/"); + + // Use URI.create to pass a pre-encoded URL — get(String) would double-encode CJK sequences + getClient().perform(get(URI.create(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + + "/%E6%97%A5%E6%9C%AC%E8%AA%9E.txt"))) + .andExpect(status().isOk()) + .andExpect(header().string(HttpHeaders.CONTENT_DISPOSITION, + // CJK chars replaced with _ in ASCII fallback; filename* has UTF-8 encoding + equalTo("attachment; filename=\"___.txt\"; " + + "filename*=UTF-8''%E6%97%A5%E6%9C%AC%E8%AA%9E.txt"))) + .andExpect(content().string(bitstreamContent)); + } + + @Test + public void downloadBitstreamByHandleSameNameDifferentBundles() throws Exception { + // A file with the same name in ORIGINAL and TEXT bundles — only ORIGINAL should be served + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + Item item = ItemBuilder.createItem(context, col) + .withAuthor("Test Author") + .build(); + String originalContent = "OriginalBundleContent"; + try (InputStream is = IOUtils.toInputStream(originalContent, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, item, is) + .withName("data.txt") + .withMimeType("text/plain") + .build(); + } + // Add same name in TEXT bundle + Bundle textBundle = BundleBuilder.createBundle(context, item) + .withName("TEXT") + .build(); + String textContent = "TextBundleContent"; + try (InputStream is = IOUtils.toInputStream(textContent, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, textBundle, is) + .withName("data.txt") + .withMimeType("text/plain") + .build(); + } + context.restoreAuthSystemState(); + + String handle = item.getHandle(); + String[] handleParts = handle.split("/"); + + // Should return ORIGINAL bundle content, not TEXT bundle + getClient().perform(get(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + "/data.txt")) + .andExpect(status().isOk()) + .andExpect(content().string(originalContent)); + } + + @Test + public void downloadBitstreamByHandleComplexFilename() throws Exception { + // Verify a filename with diacritics, plus, hash, and unmatched parenthesis + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + Item item = ItemBuilder.createItem(context, col) + .withAuthor("Test Author") + .build(); + // "M\u00e9di\u00e1 (+)#9) ano" + String complexName = "M\u00e9di\u00e1 (+)#9) ano"; + String bitstreamContent = "ComplexNameContent"; + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + BitstreamBuilder.createBitstream(context, item, is) + .withName(complexName) + .withMimeType("application/octet-stream") + .build(); + } + context.restoreAuthSystemState(); + + String handle = item.getHandle(); + String[] handleParts = handle.split("/"); + + // Pre-encoded URL: e=C3A9, a=C3A1, space=20, (=28, +=2B, )=29, #=23 + getClient().perform(get(URI.create(ENDPOINT_BASE + "/" + handleParts[0] + "/" + handleParts[1] + + "/M%C3%A9di%C3%A1%20(%2B)%239)%20ano"))) + .andExpect(status().isOk()) + .andExpect(header().string(HttpHeaders.CONTENT_DISPOSITION, + equalTo("attachment; filename=\"M_di_ (+)#9) ano\"; " + + "filename*=UTF-8''M%C3%A9di%C3%A1%20%28%2B%29%239%29%20ano"))) + .andExpect(content().string(bitstreamContent)); + } +} From 1dc5339b785c5710aef2bfe57952f20ae1768724 Mon Sep 17 00:00:00 2001 From: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> Date: Fri, 13 Mar 2026 15:00:08 +0100 Subject: [PATCH 08/41] Reduce warn logs noise (#1268) * Log 404 responses at DEBUG instead of WARN to reduce log noise * Log 404 responses at DEBUG instead of WARN (configurable via logging.server.debug-404) * Skip stack trace extraction for suppressed 404 debug logs * Replace custom debug-404 property with dedicated Log4j2 logger (org.dspace.app.rest.NotFound) * Suppress 404 warn logs via dedicated Log4j2 logger (org.dspace.app.rest.NotFound) * Turn off that warn logs for the dspace.log * Updated log name to be more unique --- .../DSpaceApiExceptionControllerAdvice.java | 31 ++++++++++++++----- dspace/config/log4j2.xml | 7 +++++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/DSpaceApiExceptionControllerAdvice.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/DSpaceApiExceptionControllerAdvice.java index 4833cb938317..7faf82f379f4 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/DSpaceApiExceptionControllerAdvice.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/DSpaceApiExceptionControllerAdvice.java @@ -61,6 +61,13 @@ public class DSpaceApiExceptionControllerAdvice extends ResponseEntityExceptionHandler { private static final Logger log = LogManager.getLogger(); + /** + * Dedicated logger for 404 NOT_FOUND responses. Configured at OFF level by default + * so that expected 404s don't flood production logs. + * Set to WARN in log4j2.xml to see 404 responses in logs. + */ + private static final Logger notFoundLog = LogManager.getLogger("org.dspace.app.rest.exception.DSpaceApiExceptionControllerAdvice.NotFound"); + /** * Default collection of HTTP error codes to log as ERROR with full stack trace. */ @@ -283,11 +290,9 @@ private void sendErrorResponseFromException(final HttpServletRequest request, if (statusCodesLoggedAsErrors.contains(statusCode)) { log.error("{} (status:{})", message, statusCode, ex); } else { - // Log the error as a single-line WARN StackTraceElement[] trace = ex.getStackTrace(); String location = trace.length <= 0 ? "unknown" : trace[0].toString(); - log.warn("{} (status:{} exception: {} at: {})", - message, statusCode, ex.getClass().getName(), location); + logClientError(statusCode, message, ex.getClass().getName(), location); } response.sendError(statusCode, message); @@ -322,7 +327,6 @@ private void sendErrorResponse(final HttpServletRequest request, // Log the full error and status code log.error("{} (status:{})", message, statusCode, ex); } else if (HttpStatus.valueOf(statusCode).is4xxClientError()) { - // Log the error as a single-line WARN String location; String exceptionMessage; if (null == ex) { @@ -333,14 +337,28 @@ private void sendErrorResponse(final HttpServletRequest request, StackTraceElement[] trace = ex.getStackTrace(); location = trace.length <= 0 ? "unknown" : trace[0].toString(); } - log.warn("{} (status:{} exception: {} at: {})", message, statusCode, - exceptionMessage, location); + logClientError(statusCode, message, exceptionMessage, location); } //Exception properties will be set by org.springframework.boot.web.support.ErrorPageFilter response.sendError(statusCode, message); } + /** + * Log a 4xx client error. 404 NOT_FOUND is sent to a dedicated logger ({@link #notFoundLog}) + * at WARN level, but the logger is set to OFF by default in log4j2.xml (suppressed). + * Set logger to WARN in log4j2.xml to see 404 responses in logs. + */ + private void logClientError(int statusCode, String message, String exceptionMessage, String location) { + if (statusCode == HttpServletResponse.SC_NOT_FOUND) { + notFoundLog.warn("{} (status:{} exception: {} at: {})", message, statusCode, + exceptionMessage, location); + } else { + log.warn("{} (status:{} exception: {} at: {})", message, statusCode, + exceptionMessage, location); + } + } + /** * Get set of status codes that should be treated as errors. * @@ -355,7 +373,6 @@ private Set getStatusCodesLoggedAsErrors() { statusCodesLoggedAsErrors.add(Integer.valueOf(code)); } catch (NumberFormatException e) { log.warn("Non-integer HTTP status code {} in {}", code, P_LOG_AS_ERROR); - // And continue } } return statusCodesLoggedAsErrors; diff --git a/dspace/config/log4j2.xml b/dspace/config/log4j2.xml index 3273551bc0f6..a2ad06ee33a6 100644 --- a/dspace/config/log4j2.xml +++ b/dspace/config/log4j2.xml @@ -89,6 +89,13 @@ + + + + + Date: Fri, 13 Mar 2026 15:24:27 +0100 Subject: [PATCH 09/41] The row lenght was updated to be less than 120 chars (#1274) --- .../app/rest/exception/DSpaceApiExceptionControllerAdvice.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/DSpaceApiExceptionControllerAdvice.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/DSpaceApiExceptionControllerAdvice.java index 7faf82f379f4..ce05086ab96b 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/DSpaceApiExceptionControllerAdvice.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/DSpaceApiExceptionControllerAdvice.java @@ -66,7 +66,8 @@ public class DSpaceApiExceptionControllerAdvice extends ResponseEntityExceptionH * so that expected 404s don't flood production logs. * Set to WARN in log4j2.xml to see 404 responses in logs. */ - private static final Logger notFoundLog = LogManager.getLogger("org.dspace.app.rest.exception.DSpaceApiExceptionControllerAdvice.NotFound"); + private static final Logger notFoundLog = + LogManager.getLogger("org.dspace.app.rest.exception.DSpaceApiExceptionControllerAdvice.NotFound"); /** * Default collection of HTTP error codes to log as ERROR with full stack trace. From 84e9f3a2c2910f035c03b4eac6e13169cab4a6e3 Mon Sep 17 00:00:00 2001 From: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> Date: Fri, 13 Mar 2026 16:20:28 +0100 Subject: [PATCH 10/41] Reduce noisy WARN logs to DEBUG level (#1269) Changed two frequently occurring WARN log messages to DEBUG level: - Context.java: 'Initializing a context while an active transaction exists' - ClarinItemServiceImpl.java: 'Cannot update item dates metadata because the approximate date is empty' --- .../java/org/dspace/content/clarin/ClarinItemServiceImpl.java | 2 +- dspace-api/src/main/java/org/dspace/core/Context.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/clarin/ClarinItemServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/clarin/ClarinItemServiceImpl.java index 39495fed64a0..018964b4cbf0 100644 --- a/dspace-api/src/main/java/org/dspace/content/clarin/ClarinItemServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/clarin/ClarinItemServiceImpl.java @@ -227,7 +227,7 @@ public void updateItemDatesMetadata(Context context, Item item) throws SQLExcept itemService.getMetadata(item, "local", "approximateDate", "issued", Item.ANY, false); if (CollectionUtils.isEmpty(approximatedDates) || StringUtils.isBlank(approximatedDates.get(0).getValue())) { - log.warn("Cannot update item dates metadata because the approximate date is empty."); + log.debug("Cannot update item dates metadata because the approximate date is empty."); return; } diff --git a/dspace-api/src/main/java/org/dspace/core/Context.java b/dspace-api/src/main/java/org/dspace/core/Context.java index c482f1de2c30..e721deff5e71 100644 --- a/dspace-api/src/main/java/org/dspace/core/Context.java +++ b/dspace-api/src/main/java/org/dspace/core/Context.java @@ -187,8 +187,8 @@ protected void init() { "Check previous entries in the dspace.log to find why the db failed to initialize."); } else { if (isTransactionAlive()) { - log.warn("Initializing a context while an active transaction exists. Context with hash: {}.", - getHash()); + log.debug("Initializing a context while an active transaction exists. Context with hash: {}.", + getHash()); } } } From cf264f7bbb078be816581dbf93f0efc5268512bd Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Tue, 21 Apr 2026 14:41:30 +0200 Subject: [PATCH 11/41] Added oai bundle exclude feature --- .../java/org/dspace/xoai/util/ItemUtils.java | 43 ++++ .../app/oai/OAIPMHBundleExposureIT.java | 188 ++++++++++++++++++ dspace/config/modules/oai.cfg | 9 + 3 files changed, 240 insertions(+) create mode 100644 dspace-server-webapp/src/test/java/org/dspace/app/oai/OAIPMHBundleExposureIT.java diff --git a/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java b/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java index 78f4571b6216..edf27dfccd8e 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java @@ -11,7 +11,9 @@ import java.io.IOException; import java.io.InputStream; import java.sql.SQLException; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import com.lyncode.xoai.dataprovider.xml.xoai.Element; @@ -107,14 +109,55 @@ public static Element.Field createValue(String name, String value) { return e; } + /** + * Default list of bundle names that must never be exposed through OAI-PMH. + * These are typically derivative bundles produced by {@code dspace filter-media} + * (extracted plain-text for indexing, generated thumbnails) or internal bundles + * such as the SWORD deposit package. Exposing them leaks content that is not + * intended to be a first-class resource of the item (see ufal/clarin-dspace#1355). + * The list is overridable through the {@code oai.bundle.excluded} configuration + * property (comma separated list of bundle names). + */ + private static final String[] DEFAULT_EXCLUDED_BUNDLES = new String[] { + "TEXT", "THUMBNAIL", "SWORD" + }; + + /** + * @return the names of the bundles that must not be exposed through OAI-PMH. + */ + private static Set getExcludedBundleNames() { + String[] configured = configurationService + .getArrayProperty("oai.bundle.excluded"); + String[] effective = (configured != null && configured.length > 0) + ? configured + : DEFAULT_EXCLUDED_BUNDLES; + Set excluded = new HashSet<>(); + for (String name : effective) { + if (name != null) { + String trimmed = name.trim(); + if (!trimmed.isEmpty()) { + excluded.add(trimmed); + } + } + } + return excluded; + } + private static Element createBundlesElement(Context context, Item item, AtomicBoolean restricted) throws SQLException { Element bundles = create("bundles"); List bs; + Set excludedBundleNames = getExcludedBundleNames(); + bs = item.getBundles(); for (Bundle b : bs) { + // Skip bundles that must not be exposed via OAI-PMH (e.g. TEXT/THUMBNAIL + // bundles produced by `dspace filter-media`). See ufal/clarin-dspace#1355. + if (b.getName() != null && excludedBundleNames.contains(b.getName())) { + continue; + } Element bundle = create("bundle"); bundles.getElement().add(bundle); bundle.getField().add(createValue("name", b.getName())); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/oai/OAIPMHBundleExposureIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/oai/OAIPMHBundleExposureIT.java new file mode 100644 index 000000000000..6f88aef1a6fa --- /dev/null +++ b/dspace-server-webapp/src/test/java/org/dspace/app/oai/OAIPMHBundleExposureIT.java @@ -0,0 +1,188 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.oai; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.not; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import com.lyncode.xoai.dataprovider.xml.xoai.Element; +import com.lyncode.xoai.dataprovider.xml.xoai.Metadata; +import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.content.Collection; +import org.dspace.content.Community; +import org.dspace.content.Item; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.BitstreamService; +import org.dspace.content.service.BundleService; +import org.dspace.services.ConfigurationService; +import org.dspace.xoai.util.ItemUtils; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Integration tests that verify which bundles are exposed through the XOAI + * representation used by OAI-PMH crosswalks (including the CLARIN CMDI one). + */ +public class OAIPMHBundleExposureIT extends AbstractControllerIntegrationTest { + + @Autowired + private ConfigurationService configurationService; + + private final BitstreamService bitstreamService = + ContentServiceFactory.getInstance().getBitstreamService(); + private final BundleService bundleService = + ContentServiceFactory.getInstance().getBundleService(); + + private Collection collection; + + @Before + public void setupStructure() throws Exception { + context.turnOffAuthorisationSystem(); + Community community = CommunityBuilder.createCommunity(context) + .withName("Test Community") + .build(); + collection = CollectionBuilder.createCollection(context, community) + .withName("Test Collection") + .build(); + context.restoreAuthSystemState(); + } + + /** + * Build an item that has an ORIGINAL bitstream plus the derivative/internal + * bundles that {@code dspace filter-media} / SWORD typically create. + */ + private Item buildItemWithDerivativeBundles() throws Exception { + context.turnOffAuthorisationSystem(); + Item item = ItemBuilder.createItem(context, collection) + .withTitle("Item with TEXT and THUMBNAIL bundles") + .withIssueDate("2026-01-01") + .build(); + + addBitstream(item, "ORIGINAL", "payload.pdf", "binary data"); + addBitstream(item, "TEXT", "payload.pdf.txt", "extracted text from pdf"); + addBitstream(item, "THUMBNAIL", "payload.pdf.jpg", "fake thumbnail bytes"); + addBitstream(item, "SWORD", "sword-deposit.zip", "sword payload"); + + context.restoreAuthSystemState(); + return item; + } + + private void addBitstream(Item item, String bundleName, String name, String content) + throws Exception { + org.dspace.content.Bundle bundle; + List bundles = + ContentServiceFactory.getInstance().getItemService() + .getBundles(item, bundleName); + if (bundles.isEmpty()) { + bundle = bundleService.create(context, item, bundleName); + } else { + bundle = bundles.get(0); + } + org.dspace.content.Bitstream bitstream = bitstreamService.create( + context, bundle, + new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8))); + bitstream.setName(context, name); + bitstreamService.update(context, bitstream); + } + + private List bundleNames(Metadata metadata) { + List names = new ArrayList<>(); + Element bundles = ItemUtils.getElement(metadata.getElement(), "bundles"); + if (bundles == null) { + return names; + } + for (Element bundle : bundles.getElement()) { + for (Element.Field field : bundle.getField()) { + if ("name".equals(field.getName())) { + names.add(field.getValue()); + } + } + } + return names; + } + + /** + * With the default configuration, TEXT, THUMBNAIL and SWORD bundles must be + * hidden from the XOAI document. + */ + @Test + public void defaultConfiguration_hidesFilterMediaAndSwordBundles() throws Exception { + // Ensure we rely on the built-in default; drop any stale override. + configurationService.setProperty("oai.bundle.excluded", null); + + Item item = buildItemWithDerivativeBundles(); + + Metadata metadata = ItemUtils.retrieveMetadata(context, item); + + List exposed = bundleNames(metadata); + + assertThat("ORIGINAL bundle must always be exposed via OAI-PMH", + exposed, hasItem("ORIGINAL")); + assertThat("TEXT bundle (dspace filter-media output) must not be exposed via OAI-PMH", + exposed, not(hasItem("TEXT"))); + assertThat("THUMBNAIL bundle (dspace filter-media output) must not be exposed via OAI-PMH", + exposed, not(hasItem("THUMBNAIL"))); + assertThat("SWORD bundle (internal deposit package) must not be exposed via OAI-PMH", + exposed, not(hasItem("SWORD"))); + } + + /** + * The administrator may reduce the exclusion list; when only THUMBNAIL is + * excluded, TEXT (and others) are exposed again. + */ + @Test + public void customExcludedBundles_allowsOverrideOfDefaults() throws Exception { + configurationService.setProperty("oai.bundle.excluded", "THUMBNAIL"); + try { + Item item = buildItemWithDerivativeBundles(); + Metadata metadata = ItemUtils.retrieveMetadata(context, item); + + List exposed = bundleNames(metadata); + + assertThat("With a custom exclusion list the ORIGINAL, TEXT and SWORD " + + "bundles must be exposed and only THUMBNAIL must be hidden", + exposed, + containsInAnyOrder("ORIGINAL", "TEXT", "SWORD")); + } finally { + configurationService.setProperty("oai.bundle.excluded", null); + } + } + + /** + * An empty value must fall back to the built-in defaults, otherwise a + * mis-configuration would regress to the pre-fix behaviour. + */ + @Test + public void emptyExcludedBundles_fallsBackToDefaults() throws Exception { + configurationService.setProperty("oai.bundle.excluded", ""); + try { + Item item = buildItemWithDerivativeBundles(); + Metadata metadata = ItemUtils.retrieveMetadata(context, item); + + List exposed = bundleNames(metadata); + + assertThat(exposed, hasItem("ORIGINAL")); + assertThat(exposed, not(hasItem("TEXT"))); + assertThat(exposed, not(hasItem("THUMBNAIL"))); + assertThat(exposed, not(hasItem("SWORD"))); + } finally { + configurationService.setProperty("oai.bundle.excluded", null); + } + } +} diff --git a/dspace/config/modules/oai.cfg b/dspace/config/modules/oai.cfg index b08addfda999..70817c84ff76 100644 --- a/dspace/config/modules/oai.cfg +++ b/dspace/config/modules/oai.cfg @@ -38,6 +38,15 @@ oai.solr.url=${solr.server}/${solr.multicorePrefix}oai # Base url for bitstreams oai.bitstream.baseUrl = ${dspace.ui.url} +# Bundles whose bitstreams must NEVER be exposed through OAI-PMH. +# Comma separated list of bundle names. Typical candidates are the bundles +# produced by `dspace filter-media` (TEXT plain-text extracts, THUMBNAIL +# thumbnails) and the internal SWORD deposit bundle. These are derivative / +# internal artifacts and should not appear in harvested records, e.g. in the +# lindat CMDI crosswalk. +# If this property is not set, the default value is: TEXT, THUMBNAIL, SWORD. +oai.bundle.excluded = TEXT, THUMBNAIL, SWORD + # Base Configuration Directory oai.config.dir = ${dspace.dir}/config/crosswalks/oai From 72da0c2565c16610ca725bfe47216fbbe9df3604 Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Wed, 22 Apr 2026 12:33:18 +0200 Subject: [PATCH 12/41] Updated docs --- .../main/java/org/dspace/xoai/util/ItemUtils.java | 12 +++++++----- dspace/config/modules/oai.cfg | 4 +++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java b/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java index edf27dfccd8e..00776fd2d1f1 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java @@ -110,20 +110,22 @@ public static Element.Field createValue(String name, String value) { } /** - * Default list of bundle names that must never be exposed through OAI-PMH. + * Default list of bundle names excluded from OAI-PMH exposure. * These are typically derivative bundles produced by {@code dspace filter-media} * (extracted plain-text for indexing, generated thumbnails) or internal bundles - * such as the SWORD deposit package. Exposing them leaks content that is not + * such as the SWORD deposit package. Exposing them may leak content that is not * intended to be a first-class resource of the item (see ufal/clarin-dspace#1355). - * The list is overridable through the {@code oai.bundle.excluded} configuration - * property (comma separated list of bundle names). + * The {@code oai.bundle.excluded} configuration property, when set, overrides + * this default list with a comma-separated list of bundle names. */ private static final String[] DEFAULT_EXCLUDED_BUNDLES = new String[] { "TEXT", "THUMBNAIL", "SWORD" }; /** - * @return the names of the bundles that must not be exposed through OAI-PMH. + * @return the effective names of bundles excluded from OAI-PMH exposure, + * using {@code oai.bundle.excluded} when configured, or the default + * excluded bundle list otherwise. */ private static Set getExcludedBundleNames() { String[] configured = configurationService diff --git a/dspace/config/modules/oai.cfg b/dspace/config/modules/oai.cfg index 70817c84ff76..8d9d9b1ae219 100644 --- a/dspace/config/modules/oai.cfg +++ b/dspace/config/modules/oai.cfg @@ -38,13 +38,15 @@ oai.solr.url=${solr.server}/${solr.multicorePrefix}oai # Base url for bitstreams oai.bitstream.baseUrl = ${dspace.ui.url} -# Bundles whose bitstreams must NEVER be exposed through OAI-PMH. +# Default / recommended list of bundles whose bitstreams are excluded from +# exposure through OAI-PMH. # Comma separated list of bundle names. Typical candidates are the bundles # produced by `dspace filter-media` (TEXT plain-text extracts, THUMBNAIL # thumbnails) and the internal SWORD deposit bundle. These are derivative / # internal artifacts and should not appear in harvested records, e.g. in the # lindat CMDI crosswalk. # If this property is not set, the default value is: TEXT, THUMBNAIL, SWORD. +# Changing this property changes which bundles are exposed through OAI-PMH. oai.bundle.excluded = TEXT, THUMBNAIL, SWORD # Base Configuration Directory From 8c1779fce314d8688496ee20a5ac937069ec49aa Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Wed, 22 Apr 2026 12:51:09 +0200 Subject: [PATCH 13/41] Fix OAI bundle exclusion docs and isolate OAIPMHBundleExposureIT config state --- .../app/oai/OAIPMHBundleExposureIT.java | 56 ++++++++++--------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/oai/OAIPMHBundleExposureIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/oai/OAIPMHBundleExposureIT.java index 6f88aef1a6fa..158931e49506 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/oai/OAIPMHBundleExposureIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/oai/OAIPMHBundleExposureIT.java @@ -31,6 +31,7 @@ import org.dspace.content.service.BundleService; import org.dspace.services.ConfigurationService; import org.dspace.xoai.util.ItemUtils; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -50,6 +51,7 @@ public class OAIPMHBundleExposureIT extends AbstractControllerIntegrationTest { ContentServiceFactory.getInstance().getBundleService(); private Collection collection; + private String originalOaiBundleExcluded; @Before public void setupStructure() throws Exception { @@ -61,6 +63,14 @@ public void setupStructure() throws Exception { .withName("Test Collection") .build(); context.restoreAuthSystemState(); + + // Preserve the loaded value so each test can safely mutate this property. + originalOaiBundleExcluded = configurationService.getProperty("oai.bundle.excluded"); + } + + @After + public void restoreOaiBundleExcludedConfiguration() { + configurationService.setProperty("oai.bundle.excluded", originalOaiBundleExcluded); } /** @@ -149,19 +159,16 @@ public void defaultConfiguration_hidesFilterMediaAndSwordBundles() throws Except @Test public void customExcludedBundles_allowsOverrideOfDefaults() throws Exception { configurationService.setProperty("oai.bundle.excluded", "THUMBNAIL"); - try { - Item item = buildItemWithDerivativeBundles(); - Metadata metadata = ItemUtils.retrieveMetadata(context, item); - - List exposed = bundleNames(metadata); - - assertThat("With a custom exclusion list the ORIGINAL, TEXT and SWORD " - + "bundles must be exposed and only THUMBNAIL must be hidden", - exposed, - containsInAnyOrder("ORIGINAL", "TEXT", "SWORD")); - } finally { - configurationService.setProperty("oai.bundle.excluded", null); - } + + Item item = buildItemWithDerivativeBundles(); + Metadata metadata = ItemUtils.retrieveMetadata(context, item); + + List exposed = bundleNames(metadata); + + assertThat("With a custom exclusion list the ORIGINAL, TEXT and SWORD " + + "bundles must be exposed and only THUMBNAIL must be hidden", + exposed, + containsInAnyOrder("ORIGINAL", "TEXT", "SWORD")); } /** @@ -171,18 +178,15 @@ public void customExcludedBundles_allowsOverrideOfDefaults() throws Exception { @Test public void emptyExcludedBundles_fallsBackToDefaults() throws Exception { configurationService.setProperty("oai.bundle.excluded", ""); - try { - Item item = buildItemWithDerivativeBundles(); - Metadata metadata = ItemUtils.retrieveMetadata(context, item); - - List exposed = bundleNames(metadata); - - assertThat(exposed, hasItem("ORIGINAL")); - assertThat(exposed, not(hasItem("TEXT"))); - assertThat(exposed, not(hasItem("THUMBNAIL"))); - assertThat(exposed, not(hasItem("SWORD"))); - } finally { - configurationService.setProperty("oai.bundle.excluded", null); - } + + Item item = buildItemWithDerivativeBundles(); + Metadata metadata = ItemUtils.retrieveMetadata(context, item); + + List exposed = bundleNames(metadata); + + assertThat(exposed, hasItem("ORIGINAL")); + assertThat(exposed, not(hasItem("TEXT"))); + assertThat(exposed, not(hasItem("THUMBNAIL"))); + assertThat(exposed, not(hasItem("SWORD"))); } } From 4336fb3c741765fd550d47360935b51ab92864a7 Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Wed, 22 Apr 2026 13:03:41 +0200 Subject: [PATCH 14/41] Fix indentation in OAIPMHBundleExposureIT field declaration --- .../test/java/org/dspace/app/oai/OAIPMHBundleExposureIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/oai/OAIPMHBundleExposureIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/oai/OAIPMHBundleExposureIT.java index 158931e49506..ef672380e02c 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/oai/OAIPMHBundleExposureIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/oai/OAIPMHBundleExposureIT.java @@ -51,7 +51,7 @@ public class OAIPMHBundleExposureIT extends AbstractControllerIntegrationTest { ContentServiceFactory.getInstance().getBundleService(); private Collection collection; - private String originalOaiBundleExcluded; + private String originalOaiBundleExcluded; @Before public void setupStructure() throws Exception { From 48c9da23f409529a60be66fbadc4e24739076419 Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Wed, 22 Apr 2026 14:11:46 +0200 Subject: [PATCH 15/41] Updated docs --- dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java b/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java index 00776fd2d1f1..4c3e159e8479 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java @@ -114,7 +114,7 @@ public static Element.Field createValue(String name, String value) { * These are typically derivative bundles produced by {@code dspace filter-media} * (extracted plain-text for indexing, generated thumbnails) or internal bundles * such as the SWORD deposit package. Exposing them may leak content that is not - * intended to be a first-class resource of the item (see ufal/clarin-dspace#1355). + * intended to be a first-class resource of the item. * The {@code oai.bundle.excluded} configuration property, when set, overrides * this default list with a comma-separated list of bundle names. */ @@ -156,7 +156,7 @@ private static Element createBundlesElement(Context context, Item item, AtomicBo bs = item.getBundles(); for (Bundle b : bs) { // Skip bundles that must not be exposed via OAI-PMH (e.g. TEXT/THUMBNAIL - // bundles produced by `dspace filter-media`). See ufal/clarin-dspace#1355. + // bundles produced by `dspace filter-media`). if (b.getName() != null && excludedBundleNames.contains(b.getName())) { continue; } From 843b8b051be1628f16036f077392f3477d1e1d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Tue, 5 May 2026 10:37:05 +0200 Subject: [PATCH 16/41] fix failing Curation tests (ufal/clarin-dspace#1353) (#1304) * fix RequiredMetadataIT failure * different fix for failing curator tests * change response to see last bitstream format results * cleaning custom bitstream format creation in PreviewContentServiceImplIT test * add debug messages * IIIFCacheEventConsumer: don't consume events when event subject is null (cherry picked from commit 50db8cd2a6a83f902b0eb94930bd5b543b72f21d) **NOTE**: This is without the `dspace-api/src/test/java/org/dspace/curate/ItemMetadataQACheckerIT.java` change will add that one into https://github.com/dataquest-dev/DSpace/pull/1237 Co-authored-by: Milan Kuchtiak --- .../dspace/iiif/consumer/IIIFCacheEventConsumer.java | 4 ++++ .../java/org/dspace/curate/ItemHandleCheckerIT.java | 2 ++ .../java/org/dspace/curate/RequiredMetadataIT.java | 3 +++ .../dspace/app/rest/PreviewContentServiceImplIT.java | 11 +++++++++++ 4 files changed, 20 insertions(+) diff --git a/dspace-api/src/main/java/org/dspace/iiif/consumer/IIIFCacheEventConsumer.java b/dspace-api/src/main/java/org/dspace/iiif/consumer/IIIFCacheEventConsumer.java index 1d6a6783018c..56ccf3f46161 100644 --- a/dspace-api/src/main/java/org/dspace/iiif/consumer/IIIFCacheEventConsumer.java +++ b/dspace-api/src/main/java/org/dspace/iiif/consumer/IIIFCacheEventConsumer.java @@ -124,6 +124,10 @@ public void consume(Context ctx, Event event) throws Exception { } private void addToCacheEviction(DSpaceObject subject, DSpaceObject subject2, int type) { + if (subject == null) { + log.warn("IIIF event consumer cannot evict from cache when subject is null."); + return; + } if (type == Constants.BITSTREAM) { toEvictFromCanvasCache.add(subject2); } diff --git a/dspace-api/src/test/java/org/dspace/curate/ItemHandleCheckerIT.java b/dspace-api/src/test/java/org/dspace/curate/ItemHandleCheckerIT.java index 8a7f9a8e618c..6b1df063c106 100644 --- a/dspace-api/src/test/java/org/dspace/curate/ItemHandleCheckerIT.java +++ b/dspace-api/src/test/java/org/dspace/curate/ItemHandleCheckerIT.java @@ -32,6 +32,7 @@ import org.dspace.content.service.CollectionService; import org.dspace.content.service.CommunityService; import org.dspace.content.service.ItemService; +import org.dspace.core.factory.CoreServiceFactory; import org.dspace.identifier.factory.IdentifierServiceFactory; import org.dspace.identifier.service.IdentifierService; import org.dspace.services.ConfigurationService; @@ -80,6 +81,7 @@ public class ItemHandleCheckerIT extends AbstractIntegrationTestWithDatabase { @Override public void setUp() throws Exception { super.setUp(); + CoreServiceFactory.getInstance().getPluginService().clearNamedPluginClasses(); try { //we have to create a new community in the database context.turnOffAuthorisationSystem(); diff --git a/dspace-api/src/test/java/org/dspace/curate/RequiredMetadataIT.java b/dspace-api/src/test/java/org/dspace/curate/RequiredMetadataIT.java index 9f9467b81f05..ca9be2329513 100644 --- a/dspace-api/src/test/java/org/dspace/curate/RequiredMetadataIT.java +++ b/dspace-api/src/test/java/org/dspace/curate/RequiredMetadataIT.java @@ -29,6 +29,7 @@ import org.dspace.content.service.CommunityService; import org.dspace.content.service.ItemService; import org.dspace.content.service.WorkspaceItemService; +import org.dspace.core.factory.CoreServiceFactory; import org.dspace.identifier.factory.IdentifierServiceFactory; import org.dspace.identifier.service.IdentifierService; import org.junit.After; @@ -96,6 +97,8 @@ public void setUp() throws Exception { @Test public void testPerform() throws IOException { + CoreServiceFactory.getInstance().getPluginService().clearNamedPluginClasses(); + Curator curator = new Curator(); curator.addTask(TASK_NAME); CuratorReportTest.ListReporter reporter = new CuratorReportTest.ListReporter(); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PreviewContentServiceImplIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PreviewContentServiceImplIT.java index f4dbaba3d285..1249a029845a 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PreviewContentServiceImplIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PreviewContentServiceImplIT.java @@ -27,11 +27,13 @@ import org.dspace.builder.ItemBuilder; import org.dspace.builder.PreviewContentBuilder; import org.dspace.content.Bitstream; +import org.dspace.content.BitstreamFormat; import org.dspace.content.Bundle; import org.dspace.content.Collection; import org.dspace.content.Community; import org.dspace.content.Item; import org.dspace.content.PreviewContent; +import org.dspace.content.service.BitstreamFormatService; import org.dspace.content.service.PreviewContentService; import org.dspace.util.FileInfo; import org.junit.After; @@ -44,6 +46,8 @@ public class PreviewContentServiceImplIT extends AbstractControllerIntegrationTe @Autowired PreviewContentService previewContentService; + @Autowired + BitstreamFormatService bitstreamFormatService; PreviewContent previewContent0; PreviewContent previewContent1; @@ -229,13 +233,20 @@ public void destroy() throws Exception { PreviewContentBuilder.deletePreviewContent(previewContent3.getID()); BitstreamBuilder.deleteBitstream(tarGzFile.getID()); + + BitstreamFormat customMimeTypeFormat = tarXGzipFile.getFormat(context); BitstreamBuilder.deleteBitstream(tarXGzipFile.getID()); + if (customMimeTypeFormat != null) { + bitstreamFormatService.delete(context, customMimeTypeFormat); + } + BitstreamBuilder.deleteBitstream(tgzFile.getID()); BitstreamBuilder.deleteBitstream(gzFile.getID()); BitstreamBuilder.deleteBitstream(tarXzFile.getID()); BitstreamBuilder.deleteBitstream(xzFile.getID()); BitstreamBuilder.deleteBitstream(tarGzFileWithWrongExtension.getID()); BitstreamBuilder.deleteBitstream(tarXzFileWithIncorrectMimeType.getID()); + super.destroy(); } From 016539acad4234da15d6a67bf6f438dff2bd0ba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Tue, 5 May 2026 10:42:06 +0200 Subject: [PATCH 17/41] UFAL/Remove 'clariah' submission process (#1305) Removed the 'clariah' submission process it is not used (cherry picked from commit cae96d523a9ab5fa866e43adcc90ca29afdcb5a5) --- dspace/config/item-submission.xml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/dspace/config/item-submission.xml b/dspace/config/item-submission.xml index ab9579f759f4..f51eb99fb497 100644 --- a/dspace/config/item-submission.xml +++ b/dspace/config/item-submission.xml @@ -448,17 +448,6 @@ - - - - - - - - - - - From 68463d490ffe385e7090b445951361425b90ce00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Tue, 5 May 2026 10:54:41 +0200 Subject: [PATCH 18/41] UFAL/Issue 1349: admin user is not allowed to delete himself/herself (ufal/clarin-dspace#1350) (#1306) * Issue 1349: admin user is not allowed to delete himself/herself * improve the fix: test context.getCurrentUser() for null * throw IllegalStateException rather than AuthorizeException, and allow client to see the error message (cherry picked from commit b913627804b3f58c50788139438a7fef536c6841) Co-authored-by: Milan Kuchtiak --- .../org/dspace/eperson/EPersonServiceImpl.java | 8 ++++++++ .../app/rest/repository/EPersonRestRepository.java | 2 +- .../dspace/app/rest/EPersonRestRepositoryIT.java | 14 ++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/eperson/EPersonServiceImpl.java b/dspace-api/src/main/java/org/dspace/eperson/EPersonServiceImpl.java index 453d5d0726be..4370ff4998fd 100644 --- a/dspace-api/src/main/java/org/dspace/eperson/EPersonServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/eperson/EPersonServiceImpl.java @@ -376,6 +376,14 @@ public void delete(Context context, EPerson ePerson, boolean cascade) throw new AuthorizeException( "You must be an admin to delete an EPerson"); } + // Admin cannot delete himself/herself + if (!context.ignoreAuthorization()) { + EPerson currentUser = context.getCurrentUser(); + if (currentUser != null && ePerson.getID().equals(currentUser.getID())) { + throw new IllegalStateException( + "You, as admin user, cannot delete yourself"); + } + } // Get all workflow-related groups that the current EPerson belongs to Set workFlowGroups = getAllWorkFlowGroups(context, ePerson); for (Group group: workFlowGroups) { diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/EPersonRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/EPersonRestRepository.java index b213b6d1eabb..1d4b95459647 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/EPersonRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/EPersonRestRepository.java @@ -375,7 +375,7 @@ protected void delete(Context context, UUID id) throws AuthorizeException { } catch (EmptyWorkflowGroupException e) { throw new RESTEmptyWorkflowGroupException(e); } catch (IllegalStateException e) { - throw new UnprocessableEntityException(e.getMessage(), e); + throw new DSpaceBadRequestException(e.getMessage(), e); } } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/EPersonRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/EPersonRestRepositoryIT.java index e1febcfa0fe0..7dc817a5cdc2 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/EPersonRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/EPersonRestRepositoryIT.java @@ -1126,6 +1126,20 @@ public void deleteForbidden() throws Exception { .andExpect(status().isOk()); } + @Test + public void deleteYourselfForbidden() throws Exception { + // login as admin + String adminToken = getAuthToken(admin.getEmail(), password); + + // Deleting yourself is forbidden + getClient(adminToken).perform(delete("/api/eperson/epersons/" + admin.getID())) + .andExpect(status().isBadRequest()); + + // Verify the admin is still here + getClient(adminToken).perform(get("/api/eperson/epersons/" + admin.getID())) + .andExpect(status().isOk()); + } + @Test public void deleteViolatingWorkFlowConstraints() throws Exception { // We turn off the authorization system in order to create the structure as defined below From f0ad8bf13378313648d43a6e9d7da2517e276ed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Tue, 5 May 2026 13:19:02 +0200 Subject: [PATCH 19/41] Issue 1354: add dc.relation.isreplacedby only when item is installed (ufal/clarin-dspace#1356) (#1308) * Issue 1354: add dc.relation.isreplacedby only when item is installed * set dc.relation.replaces on new item creation * fixed JavaDoc * fixed failing tests * test if dc.relation.isreplacedby metadata are only added when new item version is installed * use Context#reloadEntity rather than calling find method * removing unused field * also removing the now unused import --------- (cherry picked from commit 3e204e5e5d2267f7ac4c89d09d80930829d1cef6) Co-authored-by: Milan Kuchtiak --- .../content/InstallItemServiceImpl.java | 69 +++++++++++++++++++ ...inVersionedHandleIdentifierProviderIT.java | 15 +++- .../repository/VersionRestRepository.java | 18 +---- 3 files changed, 84 insertions(+), 18 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/InstallItemServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/InstallItemServiceImpl.java index f111892462b1..12047d8654a1 100644 --- a/dspace-api/src/main/java/org/dspace/content/InstallItemServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/InstallItemServiceImpl.java @@ -41,6 +41,9 @@ import org.dspace.services.ConfigurationService; import org.dspace.supervision.SupervisionOrder; import org.dspace.supervision.service.SupervisionOrderService; +import org.dspace.versioning.Version; +import org.dspace.versioning.service.VersionHistoryService; +import org.dspace.versioning.service.VersioningService; import org.springframework.beans.factory.annotation.Autowired; /** @@ -71,6 +74,10 @@ public class InstallItemServiceImpl implements InstallItemService { private ResourcePolicyService resourcePolicyService; @Autowired(required = true) protected ConfigurationService configurationService; + @Autowired(required = true) + protected VersioningService versioningService; + @Autowired(required = true) + protected VersionHistoryService versionHistoryService; Logger log = LogManager.getLogger(InstallItemServiceImpl.class); @@ -108,6 +115,8 @@ public Item installItem(Context c, InProgressSubmission is, // Finish up / archive the item item = finishItem(c, item, is); + fixRelationMetadata(c, item); + // As this is a BRAND NEW item, as a final step we need to remove the // submitter item policies created during deposit and replace them with // the default policies from the collection. @@ -410,4 +419,64 @@ private void createResourcePolicy(Context context, Item item, int action) throws context.restoreAuthSystemState(); } + /** + * This method adds the "dc.relation.isreplacedby" metadata field to the previous item, if exists. + * + * @param c Context + * @param item Item being installed + * @throws SQLException If there is an issue interacting with the database. + */ + private void fixRelationMetadata(Context c, Item item) throws SQLException, AuthorizeException { + String dcRelationReplaces = itemService.getMetadataFirstValue(item, "dc", "relation", "replaces", Item.ANY); + if (dcRelationReplaces == null) { + // nothing need to be done if the new item doesn't have "dc.relation.replaces" metadata field + return; + } + Version itemVersion = versioningService.getVersion(c, item); + if (itemVersion != null) { + Version previousItemVersion = + versionHistoryService.getPrevious(c, itemVersion.getVersionHistory(), itemVersion); + if (previousItemVersion != null) { + Item previousItem = previousItemVersion.getItem(); + if (previousItem != null) { + String previousIdentifierUri = + itemService.getMetadataFirstValue(previousItem, "dc", "identifier", "uri", Item.ANY); + if (dcRelationReplaces.equals(previousIdentifierUri)) { + // set "dc.relation.isreplacedby" metadata field to the previous item, + // pointing to the handle of the new item + // reload the previous item to avoid "detached entity" error + // when updating it in the setIsReplacedByMetadata() method + setIsReplacedByMetadata(c, c.reloadEntity(previousItem), item); + } + } + } + } + } + + private void setIsReplacedByMetadata(Context c, Item previousItem, Item newItem) + throws SQLException, AuthorizeException { + String identifierUri = itemService.getMetadataFirstValue(newItem, "dc", "identifier","uri", Item.ANY); + if (StringUtils.isBlank(identifierUri)) { + log.warn("The new item (id: {}) doesn't have the metadata dc.identifier.uri, " + + "so it's not possible to add dc.relation.isreplacedby to the previous item", + newItem.getID()); + } else { + boolean isReplacedByAlreadyExists = + itemService.getMetadata(previousItem, "dc", "relation", "isreplacedby", Item.ANY) + .stream() + .anyMatch(m -> identifierUri.equals(m.getValue())); + if (!isReplacedByAlreadyExists) { + itemService.addMetadata(c, previousItem, "dc", "relation", "isreplacedby", null, identifierUri); + try { + c.turnOffAuthorisationSystem(); + itemService.update(c, previousItem); + } catch (AuthorizeException e) { + throw new SQLException("Unable to update previous item after adding dc.relation.isreplacedby", e); + } finally { + c.restoreAuthSystemState(); + } + } + } + } + } diff --git a/dspace-api/src/test/java/org/dspace/identifier/ClarinVersionedHandleIdentifierProviderIT.java b/dspace-api/src/test/java/org/dspace/identifier/ClarinVersionedHandleIdentifierProviderIT.java index 355ed2a8fb90..82c78abe9628 100644 --- a/dspace-api/src/test/java/org/dspace/identifier/ClarinVersionedHandleIdentifierProviderIT.java +++ b/dspace-api/src/test/java/org/dspace/identifier/ClarinVersionedHandleIdentifierProviderIT.java @@ -40,7 +40,7 @@ /** * Unit Tests for ClarinVersionedHandleIdentifierProvider * - * @authorMilan Kuchtiak + * @author Milan Kuchtiak */ public class ClarinVersionedHandleIdentifierProviderIT extends AbstractIntegrationTestWithDatabase { private IdentifierServiceImpl identifierService; @@ -110,6 +110,11 @@ public void testNewVersionMetadata() throws Exception { assertThat(metadataValues.get(0).getValue(), equalTo(itemV1HandleRef)); WorkflowItem workflowItem = workflowItemService.create(context, itemV2, collection); + + // check dc.relation.isreplacedby metadata is not available yet on itemV1 + metadataValues = itemService.getMetadata(itemV1, "dc", "relation", "isreplacedby", Item.ANY); + assertThat(metadataValues.size(), equalTo(0)); + Item installedItem = installItemService.installItem(context, workflowItem); // get current date @@ -133,6 +138,14 @@ public void testNewVersionMetadata() throws Exception { metadataValues = itemService.getMetadata(installedItem, "dc", "identifier", "uri", Item.ANY); assertThat(metadataValues.size(), equalTo(1)); assertThat(metadataValues.get(0).getValue(), not(itemV1HandleRef)); + + // check "dc.relation.isreplacedby" metadata is set for the previous version of installedItem (itemV1) + // and points to the handle of the new version (installedItem) + String installedItemHandleRef = + itemService.getMetadataFirstValue(installedItem, "dc", "identifier", "uri", Item.ANY); + metadataValues = itemService.getMetadata(itemV1, "dc", "relation", "isreplacedby", Item.ANY); + assertThat(metadataValues.size(), equalTo(1)); + assertThat(metadataValues.get(0).getValue(), equalTo(installedItemHandleRef)); } private void registerProvider(Class type) { diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/VersionRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/VersionRestRepository.java index 7c6b15ef1430..9ac94a8ad37c 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/VersionRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/VersionRestRepository.java @@ -29,7 +29,6 @@ import org.dspace.content.service.WorkspaceItemService; import org.dspace.core.Context; import org.dspace.eperson.EPerson; -import org.dspace.handle.service.HandleService; import org.dspace.services.ConfigurationService; import org.dspace.versioning.Version; import org.dspace.versioning.VersionHistory; @@ -46,7 +45,7 @@ /** * This is the Repository that takes care of the operations on the {@link VersionRest} objects - * + * * @author Mykhaylo Boychuk (mykhaylo.boychuk at 4science.it) */ @Component(VersionRest.CATEGORY + "." + VersionRest.NAME) @@ -79,9 +78,6 @@ public class VersionRestRepository extends DSpaceRestRepository stringList) if (Objects.isNull(version)) { throw new RuntimeException("Cannot create the new version for the item with id: " + item.getID()); } - if (Objects.isNull(version.getItem())) { - throw new RuntimeException("Add metadata `dc.relation.isreplacedby` to the previous version item " + - "because the new item wasn't assigned to the version object."); - } - // Add metadata `dc.relation.isreplacedby` to the previous version item. - // The metadata value is: `dc.identifier.uri` from the new item. - String handleref = handleService.getCanonicalForm(version.getItem().getHandle()); - if (org.apache.commons.lang3.StringUtils.isBlank(handleref)) { - throw new RuntimeException("Cannot get handle in canonical form."); - } - itemService.addMetadata(context, item, "dc", "relation", "isreplacedby", null, - handleref); return converter.toRest(version, utils.obtainProjection()); } From 35e3a9bf94e045e98448f3e29f361646148fd1e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Tue, 5 May 2026 13:30:02 +0200 Subject: [PATCH 20/41] UFAL/Issue 1339: fixed NPE when hidden item metadata are checked for the item with deleted submitter (ufal/clarin-dspace#1344) (#1290) * Issue 1339: fixed NPE when hidden item metadata are checked for the item with deleted submitter (cherry picked from commit 64fda5054d9fef4eba8f7117486774677728ce4a) Co-authored-by: Milan Kuchtiak --- .../app/util/MetadataExposureServiceImpl.java | 11 +++++++---- .../org/dspace/app/rest/ItemRestRepositoryIT.java | 13 +++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/util/MetadataExposureServiceImpl.java b/dspace-api/src/main/java/org/dspace/app/util/MetadataExposureServiceImpl.java index c834e679e56e..55deff2853d2 100644 --- a/dspace-api/src/main/java/org/dspace/app/util/MetadataExposureServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/app/util/MetadataExposureServiceImpl.java @@ -21,6 +21,7 @@ import org.dspace.authorize.service.AuthorizeService; import org.dspace.content.Item; import org.dspace.core.Context; +import org.dspace.eperson.EPerson; import org.dspace.services.ConfigurationService; import org.springframework.beans.factory.annotation.Autowired; @@ -117,10 +118,12 @@ public boolean isHidden(Context context, String schema, String element, String q } // The user is not administrator, but he could be a submitter - if (hidden && Objects.nonNull(context) && Objects.nonNull(item) && - this.submitterShouldSee(schema, element, qualifier)) { - // the submitters override - hidden = !item.getSubmitter().equals(context.getCurrentUser()); + if (hidden && Objects.nonNull(context) && Objects.nonNull(item)) { + EPerson submitter = item.getSubmitter(); + if (Objects.nonNull(submitter) && this.submitterShouldSee(schema, element, qualifier)) { + // the submitters override + hidden = !submitter.equals(context.getCurrentUser()); + } } return hidden; diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ItemRestRepositoryIT.java index ec7a8272d276..14d0be8202c5 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ItemRestRepositoryIT.java @@ -5102,6 +5102,19 @@ public void submitterShouldSeeLocalNoteMetadata() throws Exception { .andExpect(jsonPath("$", HalMatcher.matchNoEmbeds())) .andExpect(jsonPath("$", existNoteLocalMetadataMatcher)) .andExpect(jsonPath("$", existDescriptionProvenanceMetadataMatcher)); + + // After the submitter is deleted, the response for the request made using the previously issued submitter + // token (which now authenticates as anonymous) should not contain + // `local.submission.note` and `dc.description.provenance` metadata + context.turnOffAuthorisationSystem(); + EPersonBuilder.deleteEPerson(submitter.getID()); + context.restoreAuthSystemState(); + + getClient(submitterToken).perform(get("/api/core/items/" + publicItem.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", HalMatcher.matchNoEmbeds())) + .andExpect(jsonPath("$", notExistNoteLocalMetadataMatcher)) + .andExpect(jsonPath("$", notExistDescriptionProvenanceMetadataMatcher)); } /** From a13cca5681d6bba7924cb8469aed3b2936b06b4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Tue, 5 May 2026 13:40:55 +0200 Subject: [PATCH 21/41] Issue 1321: disable File preview for files where user has no Bitstream READ permission (ufal/clarin-dspace#1327) (#1280) * Issue 1321: disable File preview for files where the user has no Bitstream READ permission * alow file preview in case only the License agreement is needed * don't allow to create file preview for non-authorized user, nor for item that requires license confirmation * fixed failing FilePreviewIT test. Now only the user with file READ permission can generate file preview * add more tests for HTML file preview * add test for HTML File preview * improve warning messages * extend test to see if non admin user can see already generated file preview --------- (cherry picked from commit 50d7bbc610f7ed7becb6ebae6940f447b11f7f4b) Co-authored-by: Milan Kuchtiak --- .../content/PreviewContentServiceImpl.java | 36 ++- .../scripts/filepreview/FilePreviewIT.java | 113 +++++--- .../MetadataBitstreamRestRepository.java | 10 +- .../MetadataBitstreamRestRepositoryIT.java | 244 ++++++++++++++++++ .../org/dspace/app/rest/assetstore/hello.html | 10 + 5 files changed, 362 insertions(+), 51 deletions(-) create mode 100644 dspace-server-webapp/src/test/resources/org/dspace/app/rest/assetstore/hello.html diff --git a/dspace-api/src/main/java/org/dspace/content/PreviewContentServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/PreviewContentServiceImpl.java index 793b4ffb5065..a29e05e8aefe 100644 --- a/dspace-api/src/main/java/org/dspace/content/PreviewContentServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/PreviewContentServiceImpl.java @@ -155,21 +155,26 @@ public List findAll(Context context) throws SQLException { @Override public boolean canPreview(Context context, Bitstream bitstream, boolean authorization) throws SQLException, AuthorizeException { - try { - // Check it is allowed by configuration - boolean isAllowedByCfg = configurationService.getBooleanProperty("file.preview.enabled", true); - if (!isAllowedByCfg) { - return false; - } - - // Check it is allowed by license - if (authorization) { + // Check it is allowed by configuration + boolean isAllowedByCfg = configurationService.getBooleanProperty("file.preview.enabled", true); + if (!isAllowedByCfg) { + return false; + } + if (authorization) { + // Verify that bitstream policy allows user to READ the bitstream. + // If not, the preview content is disabled. + try { authorizeService.authorizeAction(context, bitstream, Constants.READ); + } catch (AuthorizeException e) { + // In case the license agreement(for bitstream downloading) is needed, + // the MissingLicenseAgreementException, that extends AuthorizeException, is thrown. + // For this case we also disable the content preview. + // Otherwise, user could see the content of some files without accepting the agreement, + // which could cause a security issue. + return false; } - return true; - } catch (MissingLicenseAgreementException e) { - return false; } + return true; } @Override @@ -178,13 +183,16 @@ public List getFilePreviewContent(Context context, Bitstream bitstream File file = null; try { - file = bitstreamService.retrieveFile(context, bitstream, false); // Retrieve the file + file = bitstreamService.retrieveFile(context, bitstream, true); // Retrieve the file if (Objects.nonNull(file)) { fileInfos = processFileToFilePreview(context, bitstream, file); } } catch (MissingLicenseAgreementException e) { - log.error("Missing license agreement: ", e); + log.warn("File Preview disabled: Missing license agreement!"); + throw e; + } catch (AuthorizeException e) { + log.warn("File Preview disabled: Authorization error!"); throw e; } catch (IOException e) { log.error("IOException during file processing: ", e); diff --git a/dspace-api/src/test/java/org/dspace/scripts/filepreview/FilePreviewIT.java b/dspace-api/src/test/java/org/dspace/scripts/filepreview/FilePreviewIT.java index d03384c25d7b..aeb8e050e9ef 100644 --- a/dspace-api/src/test/java/org/dspace/scripts/filepreview/FilePreviewIT.java +++ b/dspace-api/src/test/java/org/dspace/scripts/filepreview/FilePreviewIT.java @@ -13,6 +13,7 @@ import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -60,7 +61,8 @@ public class FilePreviewIT extends AbstractIntegrationTestWithDatabase { Collection collection; Item item; - EPerson eperson; + // avoid using eperson created in superclass + EPerson ePerson; String PASSWORD = "test"; @Before @@ -68,7 +70,7 @@ public void setup() throws SQLException, AuthorizeException { InputStream previewZipIs = getClass().getResourceAsStream("preview-file-test.zip"); context.turnOffAuthorisationSystem(); - eperson = EPersonBuilder.createEPerson(context) + ePerson = EPersonBuilder.createEPerson(context) .withEmail("test@test.edu").withPassword(PASSWORD).build(); Community community = CommunityBuilder.createCommunity(context).withName("Com").build(); collection = CollectionBuilder.createCollection(context, community).withName("Col").build(); @@ -106,7 +108,7 @@ public void testUnauthorizedEmail() throws Exception { public void testUnauthorizedPassword() throws Exception { // Run the script TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "file-preview", "-e", eperson.getEmail()}; + String[] args = new String[] { "file-preview", "-e", ePerson.getEmail()}; int run = ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), testDSpaceRunnableHandler, kernelImpl); assertEquals(1, run); // Since a ParseException was caught, expect return code 1 @@ -116,7 +118,7 @@ public void testUnauthorizedPassword() throws Exception { public void testWhenNoFilesRun() throws Exception { TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "file-preview", "-e", eperson.getEmail(), "-p", PASSWORD }; + String[] args = new String[] { "file-preview", "-e", ePerson.getEmail(), "-p", PASSWORD }; int run = ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), testDSpaceRunnableHandler, kernelImpl); assertEquals(0, run); @@ -125,53 +127,68 @@ public void testWhenNoFilesRun() throws Exception { @Test public void testForSpecificItem() throws Exception { + Item item2 = createOtherWorkspaceItemWithBitstream(ePerson, 0); // Run the script - runScriptForItemWithBitstreams(item); + runScriptForItemWithBitstreams(item2, ePerson, PASSWORD); + + Bitstream b = bitstreamService.findAll(context).stream() + .filter(bitstream -> bitstream.getName().equals("logos.tgz")) + .findFirst().orElse(null); + + assertNotNull(b); + assertEquals("logos.tgz", b.getName()); + + // the preview content was created since the item was created by the same user as the script was run + assertTrue("Expects preview content created.", previewContentService.hasPreview(context, b)); + assertEquals(2, previewContentService.getPreview(context, b).size()); } @Test - public void testPreviewWithSyncStorage() throws Exception { - configurationService.setProperty("sync.storage.service.enabled", true); + public void testWhenScriptCannotCreateFilePreview() throws Exception { + Item item2 = createOtherWorkspaceItemWithBitstream(eperson, 0); + // Run the script as another user, without admin rights + runScriptForItemWithBitstreams(item2, ePerson, PASSWORD); - context.turnOffAuthorisationSystem(); + Bitstream b = bitstreamService.findAll(context).stream() + .filter(bitstream -> bitstream.getName().equals("logos.tgz")) + .findFirst().orElse(null); - WorkspaceItem wItem2; - try (InputStream tgzFile = getClass().getResourceAsStream("logos.tgz")) { - wItem2 = WorkspaceItemBuilder.createWorkspaceItem(context, collection) - .withBitstream("logos.tgz", "/local/path/logos.tgz", tgzFile, SYNC_STORE_NUMBER) - .build(); - } + assertNotNull(b); + assertEquals("logos.tgz", b.getName()); - context.restoreAuthSystemState(); + // the preview content cannot be created since the item was created by another user (eperson) + // than the user (ePerson) who runs the script + assertFalse("Expects preview content not created.", previewContentService.hasPreview(context, b)); - // Get the item and its bitstream - Item item2 = wItem2.getItem(); - List bundles = item2.getBundles(); - Bitstream bitstream2 = bundles.get(0).getBitstreams().get(0); + // Run the script as admin user + runScriptForItemWithBitstreams(item2, admin, password); - // Set the bitstream format to application/zip - BitstreamFormat bitstreamFormat = bitstreamFormatService.findByMIMEType(context, "application/x-gtar"); - bitstream2.setFormat(context, bitstreamFormat); - bitstreamService.update(context, bitstream2); - context.commit(); - context.reloadEntity(bitstream2); - context.reloadEntity(item2); + // now the preview content was created since the script was run by admin user + assertTrue("Expects preview content created.", previewContentService.hasPreview(context, b)); + assertEquals(2, previewContentService.getPreview(context, b).size()); + } - runScriptForItemWithBitstreams(item2); + @Test + public void testPreviewWithSyncStorage() throws Exception { + configurationService.setProperty("sync.storage.service.enabled", true); + Item item2 = createOtherWorkspaceItemWithBitstream(ePerson, SYNC_STORE_NUMBER); + // Run the script + runScriptForItemWithBitstreams(item2, ePerson, PASSWORD); - Bitstream b2 = bitstreamService.findAll(context).stream() - .filter(b -> b.getStoreNumber() == SYNC_STORE_NUMBER) + Bitstream b = bitstreamService.findAll(context).stream() + .filter(bitstream -> bitstream.getStoreNumber() == SYNC_STORE_NUMBER) .findFirst().orElse(null); - assertNotNull(b2); - assertTrue("Expects preview content created and stored.", previewContentService.hasPreview(context, b2)); + assertNotNull(b); + assertEquals("logos.tgz", b.getName()); + assertTrue("Expects preview content created and stored.", previewContentService.hasPreview(context, b)); } @Test public void testForAllItem() throws Exception { // Run the script TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "file-preview", "-e", eperson.getEmail(), "-p", PASSWORD}; + String[] args = new String[] { "file-preview", "-e", ePerson.getEmail(), "-p", PASSWORD}; int run = ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), testDSpaceRunnableHandler, kernelImpl); assertEquals(0, run); @@ -184,11 +201,11 @@ private void checkNoError(TestDSpaceRunnableHandler testDSpaceRunnableHandler) { assertThat(testDSpaceRunnableHandler.getWarningMessages(), empty()); } - private void runScriptForItemWithBitstreams(Item item) throws Exception { + private void runScriptForItemWithBitstreams(Item item, EPerson user, String password) throws Exception { // Run the script TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); String[] args = new String[] { "file-preview", "-u", item.getID().toString(), - "-e", eperson.getEmail(), "-p", PASSWORD}; + "-e", user.getEmail(), "-p", password}; int run = ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), testDSpaceRunnableHandler, kernelImpl); assertEquals(0, run); @@ -201,6 +218,32 @@ private void runScriptForItemWithBitstreams(Item item) throws Exception { assertThat(messages, hasItem(containsString("Generate the file previews for the specified item with " + "the given UUID: " + item.getID()))); assertThat(messages, - hasItem(containsString("Authentication by user: " + eperson.getEmail()))); + hasItem(containsString("Authentication by user: " + user.getEmail()))); + } + + private Item createOtherWorkspaceItemWithBitstream(EPerson user, int storageNumber) throws Exception { + context.turnOffAuthorisationSystem(); + context.setCurrentUser(user); + WorkspaceItem wItem2; + try (InputStream tgzFile = getClass().getResourceAsStream("logos.tgz")) { + wItem2 = WorkspaceItemBuilder.createWorkspaceItem(context, collection) + .withBitstream("logos.tgz", "/local/path/logos.tgz", tgzFile, storageNumber) + .build(); + } + context.restoreAuthSystemState(); + + Item item2 = wItem2.getItem(); + List bundles = item2.getBundles(); + Bitstream bitstream2 = bundles.get(0).getBitstreams().get(0); + + // Set the bitstream format to application/zip + BitstreamFormat bitstreamFormat = bitstreamFormatService.findByMIMEType(context, "application/x-gtar"); + bitstream2.setFormat(context, bitstreamFormat); + bitstreamService.update(context, bitstream2); + context.commit(); + context.reloadEntity(bitstream2); + context.reloadEntity(item2); + + return item2; } } diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataBitstreamRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataBitstreamRestRepository.java index 008635fbeeee..296dd870bf9a 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataBitstreamRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataBitstreamRestRepository.java @@ -22,6 +22,7 @@ import org.dspace.app.rest.exception.UnprocessableEntityException; import org.dspace.app.rest.model.MetadataBitstreamWrapperRest; import org.dspace.app.rest.model.wrapper.MetadataBitstreamWrapper; +import org.dspace.authorize.AuthorizeException; import org.dspace.content.Bitstream; import org.dspace.content.Bundle; import org.dspace.content.DSpaceObject; @@ -108,7 +109,7 @@ public Page findByHandle(@Parameter(value = "handl for (Bitstream bitstream : bitstreams) { String url = previewContentService.composePreviewURL(context, item, bitstream, contextPath); List fileInfos = new ArrayList<>(); - boolean canPreview = previewContentService.canPreview(context, bitstream, false); + boolean canPreview = previewContentService.canPreview(context, bitstream, true); String mimeType = bitstream.getFormat(context).getMIMEType(); // HTML content could be longer than the limit, so we do not store it in the DB. // It has to be generated even if property is false. @@ -119,7 +120,12 @@ public Page findByHandle(@Parameter(value = "handl boolean allowComposePreviewContent = configurationService.getBooleanProperty ("create.file-preview.on-item-page-load", false); if (allowComposePreviewContent) { - fileInfos.addAll(previewContentService.getFilePreviewContent(context, bitstream)); + try { + fileInfos.addAll(previewContentService.getFilePreviewContent(context, bitstream)); + } catch (AuthorizeException e) { + log.warn("Cannot create preview content for bitstream: {} because: {}", + bitstream.getID(), e.getMessage()); + } // Do not store HTML content in the database because it could be longer than the limit // of the database column if (!fileInfos.isEmpty() && diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamRestRepositoryIT.java index 544240beffcc..2b344b9862cf 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamRestRepositoryIT.java @@ -9,6 +9,7 @@ import static org.dspace.app.rest.utils.Utils.DEFAULT_PAGE_SIZE; import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.junit.Assert.assertFalse; @@ -21,29 +22,43 @@ import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.sql.SQLException; +import java.util.Set; import org.apache.commons.codec.CharEncoding; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.dspace.app.rest.test.AbstractControllerIntegrationTest; import org.dspace.app.util.Util; +import org.dspace.authorize.AuthorizeException; import org.dspace.authorize.service.AuthorizeService; import org.dspace.builder.BitstreamBuilder; +import org.dspace.builder.ClarinLicenseBuilder; +import org.dspace.builder.ClarinLicenseLabelBuilder; import org.dspace.builder.CollectionBuilder; import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.EPersonBuilder; import org.dspace.builder.ItemBuilder; import org.dspace.content.Bitstream; import org.dspace.content.Collection; import org.dspace.content.Item; +import org.dspace.content.clarin.ClarinLicense; +import org.dspace.content.clarin.ClarinLicenseLabel; +import org.dspace.content.factory.ClarinServiceFactory; import org.dspace.content.service.BundleService; import org.dspace.content.service.PreviewContentService; +import org.dspace.content.service.clarin.ClarinLicenseLabelService; import org.dspace.content.service.clarin.ClarinLicenseResourceMappingService; +import org.dspace.content.service.clarin.ClarinLicenseService; import org.dspace.core.Constants; +import org.dspace.eperson.EPerson; +import org.dspace.eperson.Group; +import org.dspace.eperson.service.GroupService; import org.dspace.services.ConfigurationService; import org.hamcrest.Matchers; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.web.servlet.MockMvc; public class MetadataBitstreamRestRepositoryIT extends AbstractControllerIntegrationTest { @@ -71,9 +86,19 @@ public class MetadataBitstreamRestRepositoryIT extends AbstractControllerIntegra @Autowired PreviewContentService previewContentService; + @Autowired + private GroupService groupService; + + EPerson ePerson; + String PASSWORD = "test"; + @Before public void setup() throws Exception { context.turnOffAuthorisationSystem(); + + ePerson = EPersonBuilder.createEPerson(context) + .withEmail("test@test.edu").withPassword(PASSWORD).build(); + parentCommunity = CommunityBuilder.createCommunity(context) .withName("Parent Community") .build(); @@ -259,6 +284,165 @@ public void searchMethodsExist() throws Exception { .andExpect(jsonPath("$._links.byHandle", notNullValue())); } + @Test + public void previewDisabledByReadPermission() throws Exception { + context.turnOffAuthorisationSystem(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity).withName("Collection2").build(); + Item item = ItemBuilder.createItem(context, col).withAuthor(AUTHOR).build(); + + try { + // create bitstream with ADMIN reader group, + // so the non admin user cannot read the bitstream and preview content is not available for non admin user + try (InputStream is = getClass().getResourceAsStream("assetstore/logos.tgz")) { + BitstreamBuilder. + createBitstream(context, item, is) + .withName("Bitstream") + .withDescription("Description") + .withMimeType("application/x-gtar") + .withReaderGroup(groupService.findByName(context, Group.ADMIN)) + .build(); + } + context.restoreAuthSystemState(); + + // Admin user can preview the archive file because the bitstream has the ADMIN read permission, + // and also the fileInfo should be generated for admin user. + checkFilePreviewAsAdmin(item, true, 2); + + // Non admin user cannot preview the archive file because the bitstream has only ADMIN read permission, + // and also the fileInfo should be empty in this case. + // Note that file preview was generated in the previous check, but it's not visible for non-authorized user. + checkFilePreview(item, false, 0); + } finally { + ItemBuilder.deleteItem(item.getID()); + CollectionBuilder.deleteCollection(col.getID()); + } + } + + @Test + public void previewDisabledForHtmlFileByReadPermission() throws Exception { + context.turnOffAuthorisationSystem(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity).withName("Collection2").build(); + Item item = ItemBuilder.createItem(context, col).withAuthor(AUTHOR).build(); + + try { + // create bitstream with ADMIN reader group, + // so the non admin user cannot read the bitstream and preview content is not available for non admin user + try (InputStream is = getClass().getResourceAsStream("assetstore/hello.html")) { + BitstreamBuilder. + createBitstream(context, item, is) + .withName("hello.html") + .withDescription("HTML file") + .withMimeType("text/html") + .withReaderGroup(groupService.findByName(context, Group.ADMIN)) + .build(); + } + context.restoreAuthSystemState(); + + // Admin user can preview the html file because the bitstream has the ADMIN read permission, + // and also the fileInfo should be generated for admin user. + checkFilePreviewAsAdmin(item, true, 1); + + // Non admin user cannot preview the html file because the bitstream has only ADMIN read permission, + // and also the fileInfo should be empty in this case. + checkFilePreview(item, false, 0); + } finally { + ItemBuilder.deleteItem(item.getID()); + CollectionBuilder.deleteCollection(col.getID()); + } + } + + @Test + public void previewEnabledForHtmlFile() throws Exception { + context.turnOffAuthorisationSystem(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity).withName("Collection2").build(); + Item item = ItemBuilder.createItem(context, col).withAuthor(AUTHOR).build(); + + try { + // create bitstream with ADMIN reader group, + // so the non admin user cannot read the bitstream and preview content is not available for non admin user + try (InputStream is = getClass().getResourceAsStream("assetstore/hello.html")) { + BitstreamBuilder. + createBitstream(context, item, is) + .withName("hello.html") + .withDescription("HTML file") + .withMimeType("text/html") + .build(); + } + context.restoreAuthSystemState(); + // user can preview the html file because the bitstream has read permission + checkFilePreview(item, true, 1); + } finally { + ItemBuilder.deleteItem(item.getID()); + CollectionBuilder.deleteCollection(col.getID()); + } + } + + @Test + public void previewNotAllowedWhenClarinLicenceAgreementIsNeeded() throws Exception { + context.turnOffAuthorisationSystem(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity).withName("Collection2").build(); + Item item = ItemBuilder.createItem(context, col).withAuthor(AUTHOR).build(); + + ClarinLicenseService clarinLicenseService = ClarinServiceFactory.getInstance().getClarinLicenseService(); + ClarinLicense clarinLicense = addClarinLicenseThatNeedsConfirmation(clarinLicenseService, item); + + try { + Bitstream bitstream; + try (InputStream is = getClass().getResourceAsStream("assetstore/logos.tgz")) { + bitstream = BitstreamBuilder. + createBitstream(context, item, is) + .withName("Bitstream") + .withDescription("Description") + .withMimeType("application/x-gtar") + .build(); + } + + clarinLicenseService.addClarinLicenseToBitstream(context, item, bitstream.getBundles().get(0), bitstream); + + context.restoreAuthSystemState(); + // Non admin user cannot preview the archive file when the license agreement is needed. + checkFilePreview(item, false, 0); + } finally { + ItemBuilder.deleteItem(item.getID()); + CollectionBuilder.deleteCollection(col.getID()); + ClarinLicenseBuilder.deleteClarinLicense(clarinLicense.getID()); + ClarinLicenseLabelBuilder.deleteClarinLicenseLabel(clarinLicense.getLicenseLabels().get(0).getID()); + } + } + + @Test + public void previewNotAllowedForHtmlFileWhenLicenceAgreementIsNeeded() throws Exception { + context.turnOffAuthorisationSystem(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity).withName("Collection2").build(); + Item item = ItemBuilder.createItem(context, col).withAuthor(AUTHOR).build(); + + ClarinLicenseService clarinLicenseService = ClarinServiceFactory.getInstance().getClarinLicenseService(); + ClarinLicense clarinLicense = addClarinLicenseThatNeedsConfirmation(clarinLicenseService, item); + + try { + Bitstream bitstream; + try (InputStream is = getClass().getResourceAsStream("assetstore/hello.html")) { + bitstream = BitstreamBuilder. + createBitstream(context, item, is) + .withName("Hello.html") + .withDescription("HTML file") + .withMimeType("text/html") + .build(); + } + + clarinLicenseService.addClarinLicenseToBitstream(context, item, bitstream.getBundles().get(0), bitstream); + + context.restoreAuthSystemState(); + // Non admin user cannot preview the html file when the license agreement is needed. + checkFilePreview(item, false, 0); + } finally { + ItemBuilder.deleteItem(item.getID()); + CollectionBuilder.deleteCollection(col.getID()); + ClarinLicenseBuilder.deleteClarinLicense(clarinLicense.getID()); + ClarinLicenseLabelBuilder.deleteClarinLicenseLabel(clarinLicense.getLicenseLabels().get(0).getID()); + } + } + private void composeURL() { String identifier = null; if (publicItem != null && publicItem.getHandle() != null) { @@ -285,4 +469,64 @@ private void composeURL() { url += "&isAllowed=" + isAllowed; } + + /** + * Create a license that needs confirmation and set this license to the item, + * so the user has to confirm the license agreement before downloading the file(s). + * + * @param clarinLicenseService ClarinLicenseService + * @param item Item + * @return ClarinLicense that needs confirmation + * @throws SQLException SQLException + * @throws AuthorizeException AuthorizeException + */ + private ClarinLicense addClarinLicenseThatNeedsConfirmation(ClarinLicenseService clarinLicenseService, Item item) + throws SQLException, AuthorizeException { + + ClarinLicenseLabelService clarinLicenseLabelService = + ClarinServiceFactory.getInstance().getClarinLicenseLabelService(); + + ClarinLicenseLabel clarinLicenseLabel = ClarinLicenseLabelBuilder.createClarinLicenseLabel(context).build(); + clarinLicenseLabel.setLabel("CLL"); + clarinLicenseLabel.setTitle("CLL Title"); + clarinLicenseLabelService.update(context, clarinLicenseLabel); + + ClarinLicense clarinLicense = ClarinLicenseBuilder.createClarinLicense(context).build(); + clarinLicense.setName("CL Name"); + clarinLicense.setConfirmation(ClarinLicense.Confirmation.ASK_ALWAYS); + clarinLicense.setDefinition("CL Definition"); + clarinLicense.setRequiredInfo("CL Req"); + clarinLicense.setLicenseLabels(Set.of(clarinLicenseLabel)); + + clarinLicenseService.addLicenseMetadataToItem(context, clarinLicense, item); + clarinLicenseService.update(context, clarinLicense); + + return clarinLicense; + } + + private void checkFilePreview(Item item, boolean filePreviewExpected, int expectedFileInfoSize) throws Exception { + MockMvc client = getClient(getAuthToken(ePerson.getEmail(), PASSWORD)); + performCheck(client, item, filePreviewExpected, expectedFileInfoSize); + } + + private void checkFilePreviewAsAdmin(Item item, boolean filePreviewExpected, int expectedFileInfoSize) + throws Exception { + MockMvc client = getClient(getAuthToken(admin.getEmail(), password)); + performCheck(client, item, filePreviewExpected, expectedFileInfoSize); + } + + private void performCheck(MockMvc client, Item item, boolean filePreviewExpected, int expectedFileInfoSize) + throws Exception { + client.perform(get(METADATABITSTREAM_SEARCH_BY_HANDLE_ENDPOINT) + .param("handle", item.getHandle()) + .param("fileGrpType", FILE_GRP_TYPE)) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$._embedded.metadatabitstreams").exists()) + .andExpect(jsonPath("$._embedded.metadatabitstreams").isArray()) + .andExpect(jsonPath("$._embedded.metadatabitstreams", hasSize(1))) + .andExpect(jsonPath("$._embedded.metadatabitstreams[0].canPreview").value(filePreviewExpected)) + .andExpect(jsonPath("$._embedded.metadatabitstreams[0].fileInfo").isArray()) + .andExpect(jsonPath("$._embedded.metadatabitstreams[0].fileInfo", hasSize(expectedFileInfoSize))); + } } diff --git a/dspace-server-webapp/src/test/resources/org/dspace/app/rest/assetstore/hello.html b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/assetstore/hello.html new file mode 100644 index 000000000000..f2e613f8368b --- /dev/null +++ b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/assetstore/hello.html @@ -0,0 +1,10 @@ + + + + + Hello + + +

Hello

+ + \ No newline at end of file From 4a23b55a6b1172cabd092e68bdfe62b6f817f4b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Tue, 5 May 2026 14:02:29 +0200 Subject: [PATCH 22/41] UFAL/issue 1324: curation task - implement 3 types of reporters to allow proper report writing to selected destination (ufal/clarin-dspace#1326) (#1264) * issue 1324: implement 3 types of reporters to allow proper writing to file or to console * don't throw exception when not needed * no AbstractUnitTest required - AbstractDSpaceTest is sufficient * improve append(str) method in reporters, by using StringUtils.chomp() method (cherry picked from commit 6f19d1f54db5f729e53787d5d14ffa5f850a7a86) Co-authored-by: Milan Kuchtiak --- .../main/java/org/dspace/curate/Curation.java | 37 ++++--- .../curate/reporters/DoNothingReporter.java | 37 +++++++ .../curate/reporters/FilePrinterReporter.java | 66 +++++++++++++ .../curate/reporters/SystemOutReporter.java | 57 +++++++++++ .../dspace/curate/CuratorReporterTest.java | 97 +++++++++++++++++++ 5 files changed, 279 insertions(+), 15 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/curate/reporters/DoNothingReporter.java create mode 100644 dspace-api/src/main/java/org/dspace/curate/reporters/FilePrinterReporter.java create mode 100644 dspace-api/src/main/java/org/dspace/curate/reporters/SystemOutReporter.java create mode 100644 dspace-api/src/test/java/org/dspace/curate/CuratorReporterTest.java diff --git a/dspace-api/src/main/java/org/dspace/curate/Curation.java b/dspace-api/src/main/java/org/dspace/curate/Curation.java index 3c014fec911d..d828e3e714d8 100644 --- a/dspace-api/src/main/java/org/dspace/curate/Curation.java +++ b/dspace-api/src/main/java/org/dspace/curate/Curation.java @@ -12,10 +12,6 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.PrintStream; -import java.io.Writer; import java.sql.SQLException; import java.util.HashMap; import java.util.Iterator; @@ -23,7 +19,6 @@ import java.util.UUID; import org.apache.commons.cli.ParseException; -import org.apache.commons.io.output.NullOutputStream; import org.dspace.app.util.DSpaceObjectUtilsImpl; import org.dspace.app.util.service.DSpaceObjectUtils; import org.dspace.authorize.AuthorizeException; @@ -31,6 +26,9 @@ import org.dspace.content.factory.ContentServiceFactory; import org.dspace.core.Context; import org.dspace.core.factory.CoreServiceFactory; +import org.dspace.curate.reporters.DoNothingReporter; +import org.dspace.curate.reporters.FilePrinterReporter; +import org.dspace.curate.reporters.SystemOutReporter; import org.dspace.eperson.EPerson; import org.dspace.eperson.factory.EPersonServiceFactory; import org.dspace.eperson.service.EPersonService; @@ -53,6 +51,7 @@ public class Curation extends DSpaceRunnable { HandleService handleService = HandleServiceFactory.getInstance().getHandleService(); protected Context context; private CurationClientOptions curationClientOptions; + private Reporter outputReporter; private String task; private String taskFile; @@ -173,10 +172,20 @@ private long runQueue(TaskQueue queue, Curator curator) throws SQLException, Aut * @throws SQLException If DSpace context can't complete */ private void endScript(long timeRun) throws SQLException { - context.complete(); - if (verbose) { - long elapsed = System.currentTimeMillis() - timeRun; - this.handler.logInfo("Ending curation. Elapsed time: " + elapsed); + try { + context.complete(); + if (verbose) { + long elapsed = System.currentTimeMillis() - timeRun; + this.handler.logInfo("Ending curation. Elapsed time: " + elapsed); + } + } finally { + if (outputReporter != null) { + try { + outputReporter.close(); + } catch (Exception e) { + handler.handleException("Something went wrong trying to close the reporter", e); + } + } } } @@ -188,16 +197,14 @@ private void endScript(long timeRun) throws SQLException { */ private Curator initCurator() throws FileNotFoundException { Curator curator = new Curator(handler); - OutputStream reporterStream; if (null == this.reporter) { - reporterStream = NullOutputStream.NULL_OUTPUT_STREAM; + outputReporter = new DoNothingReporter(); } else if ("-".equals(this.reporter)) { - reporterStream = System.out; + outputReporter = new SystemOutReporter(); } else { - reporterStream = new PrintStream(this.reporter); + outputReporter = new FilePrinterReporter(this.reporter); } - Writer reportWriter = new OutputStreamWriter(reporterStream); - curator.setReporter(reportWriter); + curator.setReporter(outputReporter); if (this.scope != null) { Curator.TxScope txScope = Curator.TxScope.valueOf(this.scope.toUpperCase()); diff --git a/dspace-api/src/main/java/org/dspace/curate/reporters/DoNothingReporter.java b/dspace-api/src/main/java/org/dspace/curate/reporters/DoNothingReporter.java new file mode 100644 index 000000000000..c0dd1bf6b6e4 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/curate/reporters/DoNothingReporter.java @@ -0,0 +1,37 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.curate.reporters; + +import org.dspace.curate.Reporter; + +/** + * Reporter that ignores all input. + * + * @author Milan Kuchtiak + */ +public class DoNothingReporter implements Reporter { + + @Override + public Appendable append(CharSequence csq) { + return this; + } + + @Override + public Appendable append(CharSequence csq, int start, int end) { + return this; + } + + @Override + public Appendable append(char c) { + return this; + } + + @Override + public void close() { + } +} diff --git a/dspace-api/src/main/java/org/dspace/curate/reporters/FilePrinterReporter.java b/dspace-api/src/main/java/org/dspace/curate/reporters/FilePrinterReporter.java new file mode 100644 index 000000000000..f7f016ff6fd9 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/curate/reporters/FilePrinterReporter.java @@ -0,0 +1,66 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.curate.reporters; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; + +import org.apache.commons.lang3.StringUtils; +import org.dspace.curate.Reporter; + +/** + * Reporter that writes to a specified file. + * + * @author Milan Kuchtiak + */ +public class FilePrinterReporter implements Reporter { + private final PrintWriter writer; + + public FilePrinterReporter(String fileName) throws FileNotFoundException { + File file = new File(fileName); + try { + writer = new PrintWriter(file, StandardCharsets.UTF_8); + } catch (FileNotFoundException e) { + throw e; + } catch (IOException e) { + throw new RuntimeException("Error initializing FilePrinterReporter for file: " + fileName, e); + } + } + + @Override + public Appendable append(CharSequence csq) { + // strip newline from the end of the string to avoid double newlines when using println + // do not print empty lines + if (!StringUtils.isEmpty(csq)) { + writer.println(StringUtils.chomp(csq.toString())); + } + return this; + } + + @Override + public Appendable append(CharSequence csq, int start, int end) { + writer.append(csq, start, end); + return this; + } + + @Override + public Appendable append(char c) { + writer.append(c); + return this; + } + + @Override + public void close() { + // flush and close the writer to ensure all data is written to the file + writer.flush(); + writer.close(); + } +} diff --git a/dspace-api/src/main/java/org/dspace/curate/reporters/SystemOutReporter.java b/dspace-api/src/main/java/org/dspace/curate/reporters/SystemOutReporter.java new file mode 100644 index 000000000000..412b8ddcb557 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/curate/reporters/SystemOutReporter.java @@ -0,0 +1,57 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.curate.reporters; + +import java.io.PrintWriter; + +import org.apache.commons.lang3.StringUtils; +import org.dspace.curate.Reporter; + +/** + * Reporter that writes to console (System.out). + * + * @author Milan Kuchtiak + */ +public class SystemOutReporter implements Reporter { + + private final PrintWriter writer; + + public SystemOutReporter() { + // we use PrintWriter to avoid auto-flush after every println, + // which is the default behavior of System.out.println + writer = new PrintWriter(System.out, false); + } + + @Override + public Appendable append(CharSequence csq) { + // strip newline from the end of the string to avoid double newlines when using println + // do not print empty lines + if (!StringUtils.isEmpty(csq)) { + writer.println(StringUtils.chomp(csq.toString())); + } + return this; + } + + @Override + public Appendable append(CharSequence csq, int start, int end) { + writer.append(csq, start, end); + return this; + } + + @Override + public Appendable append(char c) { + writer.append(c); + return this; + } + + @Override + public void close() { + // Note: We don't close the PrintWriter to avoid closing System.out + writer.flush(); + } +} diff --git a/dspace-api/src/test/java/org/dspace/curate/CuratorReporterTest.java b/dspace-api/src/test/java/org/dspace/curate/CuratorReporterTest.java new file mode 100644 index 000000000000..1382d4c618bd --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/curate/CuratorReporterTest.java @@ -0,0 +1,97 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.curate; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.nio.file.Files; + +import org.dspace.AbstractDSpaceTest; +import org.dspace.content.Item; +import org.dspace.core.factory.CoreServiceFactory; +import org.dspace.ctask.general.NoOpCurationTask; +import org.dspace.curate.reporters.DoNothingReporter; +import org.dspace.curate.reporters.FilePrinterReporter; +import org.dspace.curate.reporters.SystemOutReporter; +import org.dspace.services.ConfigurationService; +import org.junit.Before; +import org.junit.Test; + +/** + * Test different Reporter implementations with Curator. + * + * @author Milan Kuchtiak + */ +public class CuratorReporterTest extends AbstractDSpaceTest { + private static final String TASK_NAME = "noop"; + private static final String TEST_HANDLE = "testHandle"; + private static final String NO_OP = "No operation performed on " + TEST_HANDLE; + + private Curator curator; + + @Before + public void setup() { + CoreServiceFactory.getInstance().getPluginService().clearNamedPluginClasses(); + + // Configure the noop task to be run. + ConfigurationService cfg = kernelImpl.getConfigurationService(); + cfg.setProperty("plugin.named.org.dspace.curate.CurationTask", + NoOpCurationTask.class.getName() + " = " + TASK_NAME); + + // Get and configure a Curator. + curator = new Curator(); + } + + @Test + public void testCurateWithDoNothingReporter() throws Exception { + try (Reporter reporter = new DoNothingReporter()) { + runCuratorWithReporter(reporter); + } + } + + @Test + public void testCurateWithSystemOutReporter() throws Exception { + try (Reporter reporter = new SystemOutReporter()) { + runCuratorWithReporter(reporter); + } + } + + @Test + public void testCurateWithFilePrinterReporter() throws Exception { + File tempFile = File.createTempFile("curator-test-report", "txt"); + try (Reporter reporter = new FilePrinterReporter(tempFile.getAbsolutePath())) { + runCuratorWithReporter(reporter); + } + // check if the file contains expected line with one line separator + String fileOutput = Files.readString(tempFile.toPath()); + assertEquals(NO_OP + System.lineSeparator(), fileOutput); + } + + @Test + public void testCurateWithStringBuilder() throws Exception { + StringBuilder stringBuilder = new StringBuilder(); + runCuratorWithReporter(stringBuilder); + assertEquals(NO_OP, stringBuilder.toString()); + } + + private void runCuratorWithReporter(Appendable reporter) throws Exception { + curator.setReporter(reporter); + curator.addTask(TASK_NAME); + Item item = mock(Item.class); + when(item.getType()).thenReturn(2); + when(item.getHandle()).thenReturn(TEST_HANDLE); + curator.curate(item); + + assertEquals(Curator.CURATE_SUCCESS, curator.getStatus(TASK_NAME)); + assertEquals(NO_OP, curator.getResult(TASK_NAME)); + } + +} From 0b5796b2964b4aa22c2a911c83603bf13a4dec73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Wed, 6 May 2026 13:06:31 +0200 Subject: [PATCH 23/41] Issue ufal/clarin-dspace#1292 link/unlink items with version relationship (ufal/clarin-dspace#1304) (#1253) * Issue 1292: script to allow link two items into version relationship * implement link and unlink actions * ItemVersionLinkerIT test * improve ItemVersionLinkerIT, fix ScriptRestRepositoryIT * improve test to be more realistic * improve options description * better call of itemService.clearMetadata() * clear correctly dc.relation.replaces and dc.relation.isreplacedby * use dc.identifier.uri metadata value rather than item.getHandle() to set dc.relation.replaces and dc.relation.isreplacedby * code-cleanup * add also as a cli script * Use Item.ANY instead of null tested with the production db dump on the items mentions in the issue (11234/1-5537). It was returning: ``` The script has started Item '11234/1-5537' has no handle assigned. ``` because it's dc.identifier.uri.* --------- (cherry picked from commit 903b35a89d01ce52c8b8a6988e7eeb89fd03e608) Co-authored-by: Milan Kuchtiak --- .../dspace/administer/ItemVersionLinker.java | 383 +++++++++++++++++ .../ItemVersionLinkerConfiguration.java | 75 ++++ .../versioning/VersionHistoryServiceImpl.java | 2 +- .../versioning/VersioningServiceImpl.java | 4 + .../versioning/service/VersioningService.java | 14 +- .../config/spring/api/scripts.xml | 7 +- .../administer/ItemVersionLinkerIT.java | 395 ++++++++++++++++++ dspace/config/spring/api/scripts.xml | 5 + dspace/config/spring/rest/scripts.xml | 5 + 9 files changed, 886 insertions(+), 4 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/administer/ItemVersionLinker.java create mode 100644 dspace-api/src/main/java/org/dspace/administer/ItemVersionLinkerConfiguration.java create mode 100644 dspace-api/src/test/java/org/dspace/administer/ItemVersionLinkerIT.java diff --git a/dspace-api/src/main/java/org/dspace/administer/ItemVersionLinker.java b/dspace-api/src/main/java/org/dspace/administer/ItemVersionLinker.java new file mode 100644 index 000000000000..ef4dacc90253 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/administer/ItemVersionLinker.java @@ -0,0 +1,383 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.administer; + +import java.sql.SQLException; +import java.util.Date; +import java.util.List; +import java.util.Objects; +import java.util.UUID; +import java.util.stream.Collectors; + +import org.apache.commons.cli.ParseException; +import org.dspace.authorize.AuthorizeException; +import org.dspace.authorize.factory.AuthorizeServiceFactory; +import org.dspace.authorize.service.AuthorizeService; +import org.dspace.content.DSpaceObject; +import org.dspace.content.Item; +import org.dspace.content.MetadataValue; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.ItemService; +import org.dspace.core.Context; +import org.dspace.eperson.EPerson; +import org.dspace.eperson.factory.EPersonServiceFactory; +import org.dspace.eperson.service.EPersonService; +import org.dspace.handle.factory.HandleServiceFactory; +import org.dspace.handle.service.HandleService; +import org.dspace.identifier.IdentifierNotFoundException; +import org.dspace.identifier.IdentifierNotResolvableException; +import org.dspace.identifier.factory.IdentifierServiceFactory; +import org.dspace.identifier.service.IdentifierService; +import org.dspace.scripts.DSpaceRunnable; +import org.dspace.scripts.configuration.ScriptConfiguration; +import org.dspace.utils.DSpace; +import org.dspace.versioning.Version; +import org.dspace.versioning.VersionHistory; +import org.dspace.versioning.factory.VersionServiceFactory; +import org.dspace.versioning.service.VersionHistoryService; +import org.dspace.versioning.service.VersioningService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This script allows to link two items into the versioning relationship, + * where the second item becomes the next version of the first item. + * + * @author Milan Kuchtiak + */ +public class ItemVersionLinker extends DSpaceRunnable { + + private static final Logger log = LoggerFactory.getLogger(ItemVersionLinker.class); + private boolean help = false; + private boolean link = false; + private String previousItemID; + private String itemID; + private String ePersonEmail; + private VersioningService versioningService; + private VersionHistoryService versionHistoryService; + private ItemService itemService; + private EPersonService ePersonService; + private IdentifierService identifierService; + private AuthorizeService authorizeService; + + /** + * This method will return the Configuration that the implementing DSpaceRunnable uses + * + * @return The {@link ScriptConfiguration} that this implementing DspaceRunnable uses + */ + @Override + public ItemVersionLinkerConfiguration getScriptConfiguration() { + return new DSpace().getServiceManager().getServiceByName("item-version-linker", + ItemVersionLinkerConfiguration.class); + } + + /** + * This method has to be included in every script and handles the setup of the script by parsing the CommandLine + * and setting the variables. + * + * @throws ParseException If something goes wrong + */ + @Override + public void setup() throws ParseException { + log.debug("Setting up {}", ItemVersionLinker.class.getName()); + + link = commandLine.hasOption("l"); + boolean unlink = commandLine.hasOption("u"); + + if (commandLine.hasOption("h") || (link && unlink) || (!link && !unlink)) { + help = true; + return; + } + + if (!commandLine.hasOption("i")) { + help = true; + return; + } + + if (link && !commandLine.hasOption("p")) { + help = true; + return; + } + + if (commandLine.hasOption("e")) { + ePersonEmail = commandLine.getOptionValue("e"); + } + + versioningService = VersionServiceFactory.getInstance().getVersionService(); + versionHistoryService = VersionServiceFactory.getInstance().getVersionHistoryService(); + itemService = ContentServiceFactory.getInstance().getItemService(); + ePersonService = EPersonServiceFactory.getInstance().getEPersonService(); + identifierService = IdentifierServiceFactory.getInstance().getIdentifierService(); + authorizeService = AuthorizeServiceFactory.getInstance().getAuthorizeService(); + } + + /** + * This method has to be included in every script and this will be the main execution block for the script that'll + * contain all the logic needed + * + * @throws Exception If something goes wrong + */ + @Override + public void internalRun() throws Exception { + log.debug("Running {}", ItemVersionLinker.class.getName()); + if (help) { + printHelp(); + return; + } + + try (Context context = new Context()) { + EPerson ePerson = getEperson(context); + if (ePerson == null) { + throw new RuntimeException("Only authenticated user can run the script."); + } + context.setCurrentUser(ePerson); + + if (ePersonEmail != null && !authorizeService.isAdmin(context)) { + handler.logError("Only admin user can run the script."); + return; + } + + itemID = commandLine.getOptionValue("i"); + Item item = findItem(context, itemID); + + if (item == null) { + throw new IllegalArgumentException(String.format("Item '%s' not found.", itemID)); + } + + if (link) { + previousItemID = commandLine.getOptionValue("p"); + Item previousItem = findItem(context, previousItemID); + if (previousItem == null) { + throw new IllegalArgumentException(String.format("Previous item '%s' not found.", previousItemID)); + } + linkItems(context, previousItem, item); + } else { + unlinkLastItem(context, item); + } + context.complete(); + } + } + + /** + * Link item with the previous item into the versioning relationship. + * + * @param context + * @param previousItem + * @param item + * @throws SQLException + * @throws AuthorizeException + */ + private void linkItems(Context context, Item previousItem, Item item) throws SQLException, AuthorizeException { + if (previousItem.getID().equals(item.getID())) { + handler.logError("Cannot create versioning relationship between the same item."); + return; + } + + if (itemService.isInProgressSubmission(context, previousItem) || + itemService.isInProgressSubmission(context, item)) { + // this script is intended to work only with archived items + handler.logError("Both items must be archived to create versioning relationship."); + return; + } + + Version previousVersion = versioningService.getVersion(context, previousItem); + + if (previousVersion != null && !isLatestVersion(context, previousVersion)) { + handler.logError(String.format("Previous item '%s' is already part of existing versioning history, " + + "and its version is not the latest version in that history.", + previousItemID)); + return; + } + + Version secondVersion = versioningService.getVersion(context, item); + if (secondVersion != null) { + // we don't allow to link item that is already part of some other versioning history + handler.logError(String.format("The item '%s' is already part of other versioning history.", itemID)); + return; + } + + String previousItemName = previousItem.getName(); + + String previousItemHandleRef = getHandleRef(previousItem); + if (previousItemHandleRef == null) { + handler.logError(getNoHandleMessage(previousItemID)); + return; + } + + String itemHandleRef = getHandleRef(item); + if (itemHandleRef == null) { + handler.logError(getNoHandleMessage(itemID)); + return; + } + + handler.logInfo(String.format("Creating versioning relationship between '%s' and '%s' items.", + previousItemID, itemID)); + + int newVersionNumber; + if (previousVersion != null) { + // create new version of item in existing versioning history + VersionHistory history = previousVersion.getVersionHistory(); + newVersionNumber = previousVersion.getVersionNumber() + 1; + versioningService.createNewVersion(context, history, item, + "Linked as the next version of " + previousItemName, new Date(), newVersionNumber); + } else { + // create new versioning history for the items + VersionHistory history = versionHistoryService.create(context); + versioningService.createNewVersion(context, history, previousItem, + "The first version of " + previousItemName, new Date(), 1); + versioningService.createNewVersion(context, history, item, + "Linked as the next version of " + previousItemName, new Date(), 2); + newVersionNumber = 2; + } + + itemService.addMetadata(context, previousItem, "dc", "relation", "isreplacedby", null, itemHandleRef); + + // remove "dc.relation.replaces" metadata, if any exists + itemService.clearMetadata(context, item, "dc", "relation", "replaces", Item.ANY); + itemService.addMetadata(context, item, "dc", "relation", "replaces", null, previousItemHandleRef); + + handler.logInfo(String.format("Item '%s' has become a new version (version %d) of item '%s'.", + itemID, newVersionNumber, previousItemID)); + } + + private void unlinkLastItem(Context context, Item item) throws SQLException, AuthorizeException { + Version version = versioningService.getVersion(context, item); + if (version == null) { + handler.logError(String.format("The item '%s', to be unlinked, is not part of any versioning history.", + itemID)); + return; + } + + if (!isLatestVersion(context, version)) { + handler.logError("Can unlink only the item whose version is the latest version in the versioning history."); + return; + } + + String itemHandleRef = getHandleRef(item); + if (itemHandleRef == null) { + handler.logError(getNoHandleMessage(itemID)); + return; + } + + handler.logInfo(String.format("Going to unlink item '%s' from the versioning history.", + itemID)); + + // remove "dc.relation.replaces" metadata, if any exists + itemService.clearMetadata(context, item, "dc", "relation", "replaces", Item.ANY); + + VersionHistory versionHistory = version.getVersionHistory(); + Version previousVersion = versionHistoryService.getPrevious(context, version.getVersionHistory(), version); + + // remove the version + versioningService.deleteVersion(context, version); + handler.logInfo(String.format("Item '%s' unlinked successfully.", itemID)); + + if (previousVersion != null) { + // from the previous item, remove the "dc.relation.isreplacedby" metadata, related to item being unlinked + List metadataValuesToRemove = + itemService.getMetadata(previousVersion.getItem(), "dc", "relation", "isreplacedby", Item.ANY) + .stream().filter(metadataValue -> itemHandleRef.equals(metadataValue.getValue())) + .collect(Collectors.toList()); + + if (!metadataValuesToRemove.isEmpty()) { + itemService.removeMetadataValues(context, previousVersion.getItem(), metadataValuesToRemove); + } + + if (isFirstVersion(context, versionHistory, previousVersion)) { + // if the previous version is the first version, we need to remove the version + // and the full versioning history as well + versioningService.deleteVersion(context, previousVersion); + versionHistoryService.delete(context, versionHistory); + + // guess identifier type for previous item (only for logging) + String previousItemID = isUUID(itemID) ? + previousVersion.getItem().getID().toString() : getHandle(previousVersion.getItem()); + + handler.logInfo(String.format("The previous item '%s' was the first version of the '%s' item, " + + "so the full versioning history associated with the items was removed as well.", + previousItemID, itemID)); + } + } else { + // there is no previous version, so we need to remove the full versioning history as well + versionHistoryService.delete(context, versionHistory); + handler.logInfo(String.format("The item '%s' had no previous version in the versioning history, " + + "so the full versioning history associated with the item was removed as well.", itemID)); + } + } + + private Item findItem(Context context, String itemId) throws SQLException { + try { + return itemService.find(context, UUID.fromString(itemId)); + } catch (IllegalArgumentException ex) { + try { + DSpaceObject dso = identifierService.resolve(context, itemId); + if (dso instanceof Item) { + return (Item) dso; + } else { + throw new IllegalArgumentException(String.format("Unable to resolve '%s' identifier.", itemId)); + } + } catch (IdentifierNotFoundException | IdentifierNotResolvableException iex) { + throw new IllegalArgumentException(iex); + } + } + } + + private EPerson getEperson(Context context) throws SQLException { + if (ePersonEmail != null) { + return ePersonService.findByEmail(context, ePersonEmail); + } else { + UUID ePersonIdentifier = getEpersonIdentifier(); + return ePersonIdentifier == null ? null : ePersonService.find(context, ePersonIdentifier); + } + } + + private boolean isLatestVersion(Context context, Version version) throws SQLException { + return versionHistoryService.isLastVersion(context, version.getVersionHistory(), version); + } + + private boolean isFirstVersion(Context context, VersionHistory versionHistory, Version version) + throws SQLException { + return versionHistoryService.isFirstVersion(context, versionHistory, version); + } + + private boolean isUUID(String itemID) { + try { + UUID.fromString(itemID); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } + + private String getHandleRef(Item item) { + return itemService.getMetadata(item, "dc", "identifier", "uri", Item.ANY) + .stream() + .findFirst() + .map(MetadataValue::getValue) + .orElse(null); + } + + private String getHandle(Item item) { + // extract handle from handle reference + // handleRef cannot be null here as this method is called only after checking for null + String handleRef = Objects.requireNonNull(getHandleRef(item)); + HandleService handleService = HandleServiceFactory.getInstance().getHandleService(); + String handlePrefix = handleService.getCanonicalPrefix(); + if (handleRef.startsWith(handlePrefix)) { + return handleRef.substring(handlePrefix.length()); + } else { + return ""; + } + } + + private static String getNoHandleMessage(String itemID) { + return String.format("Item '%s' has no handle assigned.", itemID); + } + +} + diff --git a/dspace-api/src/main/java/org/dspace/administer/ItemVersionLinkerConfiguration.java b/dspace-api/src/main/java/org/dspace/administer/ItemVersionLinkerConfiguration.java new file mode 100644 index 000000000000..a924d3de8ad0 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/administer/ItemVersionLinkerConfiguration.java @@ -0,0 +1,75 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.administer; + +import org.apache.commons.cli.Options; +import org.dspace.scripts.configuration.ScriptConfiguration; + +/** + * The {@link ScriptConfiguration} for the {@link ItemVersionLinker} script. + * + * @author Milan Kuchtiak + */ +public class ItemVersionLinkerConfiguration extends ScriptConfiguration { + + private Class dspaceRunnableClass; + + /** + * Generic getter for the dspaceRunnableClass + * + * @return the dspaceRunnableClass value of this ScriptConfiguration + */ + @Override + public Class getDspaceRunnableClass() { + return dspaceRunnableClass; + } + + /** + * Generic setter for the dspaceRunnableClass + * + * @param dspaceRunnableClass The dspaceRunnableClass to be set for this ScriptConfiguration + */ + @Override + public void setDspaceRunnableClass(Class dspaceRunnableClass) { + this.dspaceRunnableClass = dspaceRunnableClass; + } + + /** + * The getter for the options of the Script + * + * @return the options value of this ScriptConfiguration + */ + @Override + public Options getOptions() { + if (options == null) { + + Options options = new Options(); + + options.addOption("h", "help", false, "help"); + + options.addOption("l", "link", false, "link item with the previous item"); + + options.addOption("u", "unlink", false, "unlink item from the previous item in version history"); + + options.addOption("p", "previous", true, + "item handle, or UUID, of the previous(left) item that is intended to be linked with the (right)" + + " item (only required for link option)"); + + options.addOption("i", "item", true, + "item handle, or UUID, of the (right) item that is intended to be linked/unlinked with/from the " + + "previous item (required for both link and unlink options)"); + options.getOption("i").setRequired(true); + + options.addOption("e", "eperson", true, "ePerson email"); + options.getOption("e").setRequired(false); + + super.options = options; + } + return options; + } +} diff --git a/dspace-api/src/main/java/org/dspace/versioning/VersionHistoryServiceImpl.java b/dspace-api/src/main/java/org/dspace/versioning/VersionHistoryServiceImpl.java index 493861df1c60..87a865f03db7 100644 --- a/dspace-api/src/main/java/org/dspace/versioning/VersionHistoryServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/versioning/VersionHistoryServiceImpl.java @@ -74,7 +74,7 @@ public void update(Context context, List versionHistories) throw @Override public void delete(Context context, VersionHistory versionHistory) throws SQLException, AuthorizeException { - versionHistoryDAO.delete(context, new VersionHistory()); + versionHistoryDAO.delete(context, versionHistory); } // LIST order: descending diff --git a/dspace-api/src/main/java/org/dspace/versioning/VersioningServiceImpl.java b/dspace-api/src/main/java/org/dspace/versioning/VersioningServiceImpl.java index b6f708b50066..9394d72498bc 100644 --- a/dspace-api/src/main/java/org/dspace/versioning/VersioningServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/versioning/VersioningServiceImpl.java @@ -266,4 +266,8 @@ public int countVersionsByHistoryWithItem(Context context, VersionHistory versio return versionDAO.countVersionsByHistoryWithItem(context, versionHistory); } + @Override + public void deleteVersion(Context c, Version version) throws SQLException { + versionDAO.delete(c, version); + } } diff --git a/dspace-api/src/main/java/org/dspace/versioning/service/VersioningService.java b/dspace-api/src/main/java/org/dspace/versioning/service/VersioningService.java index 2f6df5b732f7..a34a8e6bd43e 100644 --- a/dspace-api/src/main/java/org/dspace/versioning/service/VersioningService.java +++ b/dspace-api/src/main/java/org/dspace/versioning/service/VersioningService.java @@ -45,7 +45,7 @@ public interface VersioningService { * To keep version numbers stable we do not delete versions, we do only set * the item, date, summary and eperson null. This methods returns only those * versions that have an item assigned. - * + * * @param c The relevant DSpace Context. * @param vh Version history * @param offset The position of the first result to return @@ -79,6 +79,16 @@ List getVersionsByHistoryWithItems(Context c, VersionHistory vh, int of Version createNewVersion(Context context, VersionHistory history, Item item, String summary, Date date, int versionNumber); + /** + * This method deletes the version associated with the item from the versioning history, + * but unlike the {@link #delete} method this doesn't delete the entire item. + * + * @param c context + * @param version version + * @throws SQLException if database error + */ + void deleteVersion(Context c, Version version) throws SQLException; + /** * Update the Version * @@ -94,7 +104,7 @@ Version createNewVersion(Context context, VersionHistory history, Item item, Str * remove a version we set the item, date, summary and eperson null. This * method returns only versions that aren't soft deleted and have items * assigned. - * + * * @param context The relevant DSpace Context. * @param versionHistory Version history * @return Total versions of an version history that have items assigned. diff --git a/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml b/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml index 95d34d77c7cc..aaecc074e0bb 100644 --- a/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml +++ b/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml @@ -39,7 +39,7 @@ - + @@ -106,6 +106,11 @@ + + + + + diff --git a/dspace-api/src/test/java/org/dspace/administer/ItemVersionLinkerIT.java b/dspace-api/src/test/java/org/dspace/administer/ItemVersionLinkerIT.java new file mode 100644 index 000000000000..a910c992e186 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/administer/ItemVersionLinkerIT.java @@ -0,0 +1,395 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.administer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.sql.SQLException; +import java.util.Date; +import java.util.List; +import java.util.UUID; + +import org.dspace.AbstractIntegrationTestWithDatabase; +import org.dspace.app.launcher.ScriptLauncher; +import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.content.Collection; +import org.dspace.content.Community; +import org.dspace.content.Item; +import org.dspace.content.MetadataValue; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.ItemService; +import org.dspace.eperson.EPerson; +import org.dspace.handle.factory.HandleServiceFactory; +import org.dspace.handle.service.HandleService; +import org.dspace.versioning.Version; +import org.dspace.versioning.VersionHistory; +import org.dspace.versioning.factory.VersionServiceFactory; +import org.dspace.versioning.service.VersionHistoryService; +import org.dspace.versioning.service.VersioningService; +import org.junit.Before; +import org.junit.Test; + +public class ItemVersionLinkerIT extends AbstractIntegrationTestWithDatabase { + + private TestDSpaceRunnableHandler testDSpaceRunnableHandler; + private Collection collection; + private Item item1; + private Item item2; + private Item item3; + + private ItemService itemService; + private VersioningService versioningService; + private VersionHistoryService versionHistoryService; + private HandleService handleService; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + context.setCurrentUser(admin); + Community community = CommunityBuilder.createCommunity(context).build(); + collection = CollectionBuilder.createCollection(context, community) + .withSubmitterGroup(eperson) + .build(); + item1 = ItemBuilder.createItem(context, collection).withTitle("Item 1").build(); + item2 = ItemBuilder.createItem(context, collection).withTitle("Item 2").build(); + item3 = ItemBuilder.createItem(context, collection).withTitle("Item 3").build(); + itemService = ContentServiceFactory.getInstance().getItemService(); + versioningService = VersionServiceFactory.getInstance().getVersionService(); + versionHistoryService = VersionServiceFactory.getInstance().getVersionHistoryService(); + handleService = HandleServiceFactory.getInstance().getHandleService(); + testDSpaceRunnableHandler = createTestHandler(); + } + + @Test() + public void testLink() throws Exception { + // link item1 with item2 should pass + runScript(getLinkOptions(item1, item2, admin)); + assertLinkMessages(item1, item2, 2); + testDSpaceRunnableHandler.getInfoMessages().clear(); + + // linking item1 with item3 should fail since item1 is not the last version anymore + runScript(getLinkOptions(item1, item3, admin)); + assertEquals(1, testDSpaceRunnableHandler.getErrorMessages().size()); + assertEquals(String.format("Previous item '%s' is already part of existing versioning history, " + + "and its version is not the latest version in that history.", item1.getID()), getErrorMessage()); + testDSpaceRunnableHandler.getErrorMessages().clear(); + + // there is a limitation that an item that is going to be connected with previous item + // cannot be part of any versioning history (we don't support one item being part of two versioning histories) + // in this case, item2 is already in version history with item1 + runScript(getLinkOptions(item3, item2, admin)); + assertEquals(1, testDSpaceRunnableHandler.getErrorMessages().size()); + assertEquals(getLinkErrorMessagePartOfOtherVersionHistory(item2), getErrorMessage()); + testDSpaceRunnableHandler.getErrorMessages().clear(); + + // linking item3 with item1 should fail (same as above) + runScript(getLinkOptions(item3, item1, admin)); + assertEquals(1, testDSpaceRunnableHandler.getErrorMessages().size()); + assertEquals(getLinkErrorMessagePartOfOtherVersionHistory(item1), getErrorMessage()); + testDSpaceRunnableHandler.getErrorMessages().clear(); + + // linking item2 with item1 should fail also (cyclic linking) + runScript(getLinkOptions(item2, item1, admin)); + assertEquals(1, testDSpaceRunnableHandler.getErrorMessages().size()); + assertEquals(getLinkErrorMessagePartOfOtherVersionHistory(item1), getErrorMessage()); + } + + @Test() + public void testLink3Items() throws Exception { + // create version history with item1 and item2 + VersionHistory versionHistory = versionHistoryService.create(context); + createNewVersion(versionHistory, item1, 1); + createNewVersion(versionHistory, item2, 2); + + // link item2 with item3 should pass + runScript(getLinkOptions(item2, item3, admin)); + assertLinkMessages(item2, item3, 3); + + Version v3 = versioningService.getVersion(context, item3); + assertEquals(v3.getVersionHistory(), versionHistory); + } + + @Test() + public void testLinkErrorNotAdmin() throws Exception { + runScript(getLinkOptions(item1, item2, eperson)); + assertEquals(1, testDSpaceRunnableHandler.getErrorMessages().size()); + assertEquals("Only admin user can run the script.", getErrorMessage()); + } + + @Test() + public void testLinkErrorItemToItself() throws Exception { + runScript(getLinkOptions(item1, item1, admin)); + assertEquals(1, testDSpaceRunnableHandler.getErrorMessages().size()); + assertEquals("Cannot create versioning relationship between the same item.", getErrorMessage()); + } + + @Test() + public void testLinkItemNoHandle() throws Exception { + Item item4 = ItemBuilder.createItem(context, collection).withTitle("Item 4").build(); + itemService.clearMetadata(context, item4, "dc", "identifier", "uri", Item.ANY); + + // linking item1 with item4 should fail since item4 has no handle + runScript(getLinkOptions(item1, item4, admin)); + assertEquals(1, testDSpaceRunnableHandler.getErrorMessages().size()); + assertEquals(getNoHandleMessage(item4.getID()), getErrorMessage()); + testDSpaceRunnableHandler.getErrorMessages().clear(); + + // linking item4 with item1 should also fail since item4 has no handle + runScript(getLinkOptions(item4, item1, admin)); + assertEquals(1, testDSpaceRunnableHandler.getErrorMessages().size()); + assertEquals(getNoHandleMessage(item4.getID()), getErrorMessage()); + } + + @Test() + public void testLinkErrorInvalidUuid() throws Exception { + runScript(new String[]{"item-version-linker", "-l", "-p", item1.getHandle(), + "-i", "invalid-uuid", "-e", admin.getEmail()}); + assertNotNull(testDSpaceRunnableHandler.getException()); + assertEquals("Unable to resolve 'invalid-uuid' identifier.", getExceptionMessage()); + } + + @Test() + public void testLinkErrorItemNotFound() throws Exception { + UUID randomUUID = UUID.randomUUID(); + runScript(new String[] { "item-version-linker", "-l", "-p", item1.getHandle(), + "-i", randomUUID.toString(), "-e", admin.getEmail() }); + assertNotNull(testDSpaceRunnableHandler.getException()); + assertEquals(String.format("Item '%s' not found.", randomUUID), getExceptionMessage()); + } + + @Test() + public void testUnlink() throws Exception { + // create version history with item1, item2 and item3 + VersionHistory versionHistory = versionHistoryService.create(context); + createNewVersion(versionHistory, item1, 1); + createNewVersion(versionHistory, item2, 2); + createNewVersion(versionHistory, item3, 3); + + // unlinking item3 + runScript(getUnlinkOptions(item3, admin)); + assertUnlinkMessages(item2, item3, item3.getID()); + testDSpaceRunnableHandler.getInfoMessages().clear(); + + // unlinking item3 again should fail as item3 is not linked anymore + runScript(getUnlinkOptions(item3, admin)); + assertEquals(getUnlinkErrorMessageNotPartOfVersionHistory(item3), getErrorMessage()); + testDSpaceRunnableHandler.getErrorMessages().clear(); + + // unlinking item1 should fail as item1 is not the latest version + runScript(getUnlinkOptions(item1, admin)); + assertEquals(getUnlinkErrorMessageNotLastItem(), getErrorMessage()); + testDSpaceRunnableHandler.getErrorMessages().clear(); + } + + @Test() + public void testUnlinkLastItems() throws Exception { + // create version history with item1 and item2 + VersionHistory versionHistory = versionHistoryService.create(context); + createNewVersion(versionHistory, item1, 1); + createNewVersion(versionHistory, item2, 2); + + // unlinking item2 (will unlink both item1 and item2 since item1 was the first version) + runScript(getUnlinkOptions(item2, admin)); + assertUnlinkMessagesLastItems(item1, item2, item1.getID(), item2.getID()); + testDSpaceRunnableHandler.getInfoMessages().clear(); + + // unlinking item2 again should fail + runScript(getUnlinkOptions(item2, admin)); + assertEquals(getUnlinkErrorMessageNotPartOfVersionHistory(item2), getErrorMessage()); + testDSpaceRunnableHandler.getErrorMessages().clear(); + + // unlinking item1 should also fail since both items item1 and item2 were unlinked + // because item1 was the first item in the versioning history + runScript(getUnlinkOptions(item1, admin)); + assertEquals(getUnlinkErrorMessageNotPartOfVersionHistory(item1), getErrorMessage()); + + // check if version history was removed + assertNull(versionHistoryService.find(context, versionHistory.getID())); + } + + @Test() + public void testUnlinkLastItemsWithHandles() throws Exception { + // create version history with item1 and item2 + VersionHistory versionHistory = versionHistoryService.create(context); + createNewVersion(versionHistory, item1, 1); + createNewVersion(versionHistory, item2, 2); + + // unlinking item2 (will unlink both item1 and item2 since item1 was the first version) + runScript(new String[] { "item-version-linker", "-u", "-i", item2.getHandle(), "-e", admin.getEmail() }); + assertUnlinkMessagesLastItems(item1, item2, item1.getHandle(), item2.getHandle()); + + // check if version history was removed + assertNull(versionHistoryService.find(context, versionHistory.getID())); + } + + @Test() + public void testUnlinkItemNoHandle() throws Exception { + VersionHistory versionHistory = versionHistoryService.create(context); + createNewVersion(versionHistory, item1, 1); + createNewVersion(versionHistory, item2, 2); + + itemService.clearMetadata(context, item2, "dc", "identifier", "uri", Item.ANY); + + // unlinking item2 should fail since item2 has no handle + runScript(getUnlinkOptions(item2, admin)); + assertEquals(1, testDSpaceRunnableHandler.getErrorMessages().size()); + assertEquals(getNoHandleMessage(item2.getID()), getErrorMessage()); + testDSpaceRunnableHandler.getErrorMessages().clear(); + } + + @Test() + public void testUnlinkSingleItemInHistory() throws Exception { + // create version history with one item only + VersionHistory versionHistory = versionHistoryService.create(context); + createNewVersion(versionHistory, item1, 1); + + // unlinking item1 should remove also the version history since item1 is the only item in that history + runScript(getUnlinkOptions(item1, admin)); + assertEquals(0, testDSpaceRunnableHandler.getErrorMessages().size()); + List infoMessages = testDSpaceRunnableHandler.getInfoMessages(); + assertEquals(3, infoMessages.size()); + assertEquals(getUnlinkStartMessage(item1.getID()), infoMessages.get(0)); + assertEquals(getUnlinkSuccessMessage(item1.getID()), infoMessages.get(1)); + assertEquals(String.format("The item '%s' had no previous version in the versioning history, " + + "so the full versioning history associated with the item was removed as well.", item1.getID()), + infoMessages.get(2)); + testDSpaceRunnableHandler.getInfoMessages().clear(); + + // check if version history was removed + assertNull(versionHistoryService.find(context, versionHistory.getID())); + + // unlinking item1 again should fail + runScript(getUnlinkOptions(item1, admin)); + assertEquals(getUnlinkErrorMessageNotPartOfVersionHistory(item1), getErrorMessage()); + testDSpaceRunnableHandler.getErrorMessages().clear(); + } + + private void assertLinkMessages(Item item1, Item item2, int version) throws SQLException { + assertEquals(0, testDSpaceRunnableHandler.getErrorMessages().size()); + List infoMessages = testDSpaceRunnableHandler.getInfoMessages(); + assertEquals(2, infoMessages.size()); + + assertEquals(String.format("Creating versioning relationship between '%s' and '%s' items.", + item1.getID(), item2.getID()), infoMessages.get(0)); + assertEquals(String.format("Item '%s' has become a new version (version %d) of item '%s'.", + item2.getID(), version, item1.getID()), infoMessages.get(1)); + + Version v1 = versioningService.getVersion(context, item1); + Version v2 = versioningService.getVersion(context, item2); + assertEquals(v1.getVersionHistory(), v2.getVersionHistory()); + assertTrue(v1.getVersionNumber() < v2.getVersionNumber()); + + // check dc.relation metadata added + List isReplacedBy = itemService.getMetadata(item1, "dc", "relation", "isreplacedby", null); + assertEquals(1, isReplacedBy.size()); + assertTrue(isReplacedBy.get(0).getValue().endsWith(item2.getHandle())); + + List replaces = itemService.getMetadata(item2, "dc", "relation", "replaces", null); + assertEquals(1, replaces.size()); + assertTrue(replaces.get(0).getValue().endsWith(item1.getHandle())); + } + + private void assertUnlinkMessages(Item item1, Item item2, Object item2ID) { + assertEquals(0, testDSpaceRunnableHandler.getErrorMessages().size()); + List infoMessages = testDSpaceRunnableHandler.getInfoMessages(); + assertTrue(infoMessages.size() >= 2); + + assertEquals(getUnlinkStartMessage(item2ID), infoMessages.get(0)); + assertEquals(getUnlinkSuccessMessage(item2ID), infoMessages.get(1)); + + // check dc.relation metadata removed + List isReplacedBy = itemService.getMetadata(item1, "dc", "relation", "isreplacedby", null); + assertEquals(0, isReplacedBy.size()); + + List replaces = itemService.getMetadata(item2, "dc", "relation", "replaces", null); + assertEquals(0, replaces.size()); + } + + private void assertUnlinkMessagesLastItems(Item item1, Item item2, Object item1ID, Object item2ID) { + assertUnlinkMessages(item1, item2, item2ID); + assertEquals(String.format("The previous item '%s' was the first version of the '%s' item, " + + "so the full versioning history associated with the items was removed as well.", + item1ID, item2ID), + testDSpaceRunnableHandler.getInfoMessages().get(2)); + } + + private static String getUnlinkStartMessage(Object itemID) { + return String.format("Going to unlink item '%s' from the versioning history.", itemID); + } + + private static String getUnlinkSuccessMessage(Object itemID) { + return String.format("Item '%s' unlinked successfully.", itemID); + } + + private static String getNoHandleMessage(Object itemID) { + return String.format("Item '%s' has no handle assigned.", itemID); + } + + private static String getLinkErrorMessagePartOfOtherVersionHistory(Item item) { + return String.format("The item '%s' is already part of other versioning history.", item.getID()); + } + + private static String getUnlinkErrorMessageNotPartOfVersionHistory(Item item) { + return String.format("The item '%s', to be unlinked, is not part of any versioning history.", item.getID()); + } + + private static String getUnlinkErrorMessageNotLastItem() { + return "Can unlink only the item whose version is the latest version in the versioning history."; + } + + private static String[] getLinkOptions(Item item1, Item item2, EPerson eperson) { + return new String[] { "item-version-linker", + "-l", "-p", item1.getID().toString(), "-i", item2.getID().toString(), "-e", eperson.getEmail() }; + } + + private static String[] getUnlinkOptions(Item item, EPerson eperson) { + return new String[] { "item-version-linker", "-u", "-i", item.getID().toString(), "-e", eperson.getEmail() }; + } + + private void runScript(String[] args) throws Exception { + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), testDSpaceRunnableHandler, kernelImpl); + } + + private TestDSpaceRunnableHandler createTestHandler() { + return new TestDSpaceRunnableHandler(); + } + + private String getErrorMessage() { + return testDSpaceRunnableHandler.getErrorMessages().get(0); + } + + private String getExceptionMessage() { + return testDSpaceRunnableHandler.getException().getMessage(); + } + + private void createNewVersion(VersionHistory versionHistory, Item item, int versionNumber) throws SQLException { + Version version = versioningService.createNewVersion(context, versionHistory, item, + "version " + versionNumber, new Date(), versionNumber); + if (!versionHistoryService.isFirstVersion(context, versionHistory, version)) { + Version previous = versionHistoryService.getPrevious(context, versionHistory, version); + Item previousItem = previous.getItem(); + + String previousItemHandleRef = handleService.getCanonicalForm(previousItem.getHandle()); + String secondItemHandleRef = handleService.getCanonicalForm(item.getHandle()); + + itemService.addMetadata(context, previousItem, "dc", "relation", "isreplacedby", null, + secondItemHandleRef); + + itemService.addMetadata(context, item, "dc", "relation", "replaces", null, + previousItemHandleRef); + } + } + +} \ No newline at end of file diff --git a/dspace/config/spring/api/scripts.xml b/dspace/config/spring/api/scripts.xml index 336692947a2b..994621e7c9fc 100644 --- a/dspace/config/spring/api/scripts.xml +++ b/dspace/config/spring/api/scripts.xml @@ -106,4 +106,9 @@ + + + + + diff --git a/dspace/config/spring/rest/scripts.xml b/dspace/config/spring/rest/scripts.xml index 5ae2bedd7d17..0cf4d1c49ddf 100644 --- a/dspace/config/spring/rest/scripts.xml +++ b/dspace/config/spring/rest/scripts.xml @@ -94,4 +94,9 @@ + + + + + From 74957d849b8f956c413c696e156fd27537088223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Wed, 6 May 2026 16:52:41 +0200 Subject: [PATCH 24/41] UFAL/[Port to dtq-dev] Port ItemMetadataQAChecker curation task from v5 to v7 (#1237) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Port ItemMetadataQAChecker curation task from v5 to v7 (ufal/clarin-dspace#1312) * Add ItemMetadataQAChecker curation task with tests --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: kosarko <1842385+kosarko@users.noreply.github.com> (cherry picked from commit d5517bed94a4adf4b2d94c20f7b7e01e1153e432) * Issue ufal/clarin-dspace#1310 curation task to check relation metadata (ufal/clarin-dspace#1325) * issue 1310: check versioning releationship for items with relation metadata * improve implementation + test * More readable, I think. * update logging add the handle of the referenced item where possible * a test case to cover "no related item" * improve the failure message --------- Co-authored-by: Ondřej Košarko (cherry picked from commit c97f406546cb407c00beda8be740c827e4820025) * fix failing Curation tests (ufal/clarin-dspace#1353) * fix RequiredMetadataIT failure * different fix for failing curator tests * change response to see last bitstream format results * cleaning custom bitstream format creation in PreviewContentServiceImplIT test * resolve MR comments * add debug messages * more debug messages * IIIFCacheEventConsumer: don't consume events when event subject is null (cherry picked from commit 50db8cd2a6a83f902b0eb94930bd5b543b72f21d) **NOTE**: this is just `dspace-api/src/test/java/org/dspace/curate/ItemMetadataQACheckerIT.java` the rest is in dataquest-dev/dspace#1304 * removed empty line --------- Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> Co-authored-by: kosarko <1842385+kosarko@users.noreply.github.com> Co-authored-by: Milan Kuchtiak --- .../ctask/general/ItemMetadataQAChecker.java | 603 ++++++++++++++++++ .../dspaceFolder/config/modules/curate.cfg | 1 + .../curate/ItemMetadataQACheckerIT.java | 458 +++++++++++++ dspace/config/modules/curate.cfg | 1 + 4 files changed, 1063 insertions(+) create mode 100644 dspace-api/src/main/java/org/dspace/ctask/general/ItemMetadataQAChecker.java create mode 100644 dspace-api/src/test/java/org/dspace/curate/ItemMetadataQACheckerIT.java diff --git a/dspace-api/src/main/java/org/dspace/ctask/general/ItemMetadataQAChecker.java b/dspace-api/src/main/java/org/dspace/ctask/general/ItemMetadataQAChecker.java new file mode 100644 index 000000000000..c4e5ef709b67 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/ctask/general/ItemMetadataQAChecker.java @@ -0,0 +1,603 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +/* Created for LINDAT/CLARIN */ +package org.dspace.ctask.general; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.dspace.app.util.DCInput; +import org.dspace.app.util.DCInputSet; +import org.dspace.app.util.DCInputsReader; +import org.dspace.app.util.DCInputsReaderException; +import org.dspace.content.DSpaceObject; +import org.dspace.content.Item; +import org.dspace.content.MetadataValue; +import org.dspace.curate.AbstractCurationTask; +import org.dspace.curate.Curator; +import org.dspace.discovery.IsoLangCodes; +import org.dspace.versioning.VersionHistory; +import org.dspace.versioning.factory.VersionServiceFactory; +import org.dspace.versioning.service.VersionHistoryService; + +/** + * Check basic properties of item metadata for quality assurance. + * Ported from DSpace v5 CLARIN implementation. + * + * @author LINDAT/CLARIN + */ +public class ItemMetadataQAChecker extends AbstractCurationTask { + + public static final int CURATE_WARNING = -1000; + private static final Logger log = LogManager.getLogger(ItemMetadataQAChecker.class); + + /** Expected types. */ + private Set dcTypeValuesSet; + + private static final String[] rightsMdStrings = {"dc.rights.uri", "dc.rights.label", "dc.rights"}; + + private Map itemTitles; + private String handlePrefix; + private Map complexInputs; + + private String[] nonRepeatableMetadata; + private String[] strangeMetadata; + private String[] highlyRecommended; + + private VersionHistoryService versionHistoryService; + + @Override + public void init(Curator curator, String taskId) throws IOException { + super.init(curator, taskId); + itemTitles = new HashMap<>(); + handlePrefix = configurationService.getProperty("handle.canonical.prefix"); + + // Initialize expected types from configuration + String[] configuredTypes = configurationService.getArrayProperty( + "lr.curation.metadata.expected.types"); + if (configuredTypes != null && configuredTypes.length > 0) { + dcTypeValuesSet = new HashSet<>(Arrays.asList(configuredTypes)); + } else { + // Use defaults if not configured + dcTypeValuesSet = new HashSet<>(Arrays.asList( + "corpus", "lexicalConceptualResource", "languageDescription", "toolService")); + } + + nonRepeatableMetadata = configurationService.getArrayProperty("lr.curation.metadata.nonrepeatable", + new String[]{ + "local.branding", + "dc.type", + "dc.date.accessioned", + "dc.rights.label", + "dc.date.available", + "dc.source.uri", + "metashare.ResourceInfo#DistributionInfo#LicenseInfo.license" + }); + strangeMetadata = configurationService.getArrayProperty("lr.curation.metadata.strange", new String[]{ + "dc.description.uri", + }); + highlyRecommended = configurationService.getArrayProperty("lr.curation.metadata.recommended", new String[]{ + "dc.subject", + }); + + complexInputs = new HashMap<>(); + loadComplexInputs(); + + versionHistoryService = VersionServiceFactory.getInstance().getVersionHistoryService(); + } + + private void loadComplexInputs() { + try { + DCInputsReader reader = new DCInputsReader(); + // Get all input sets to check complex inputs across all forms + List inputSets = reader.getAllInputs(Integer.MAX_VALUE, 0); + + for (DCInputSet inputSet : inputSets) { + DCInput[][] fields = inputSet.getFields(); + for (DCInput[] row : fields) { + for (DCInput input : row) { + if ("complex".equals(input.getInputType())) { + String name = StringUtils.isBlank(input.getQualifier()) + ? String.format("%s.%s", input.getSchema(), input.getElement()) + : String.format("%s.%s.%s", input.getSchema(), input.getElement(), + input.getQualifier()); + complexInputs.put(name, input.getComplexDefinition().getInputs().size()); + } + } + } + } + } catch (DCInputsReaderException e) { + log.error("Problems fetching input-forms.xml", e); + } + } + + private String getHandle(Item item) { + if (null != item.getHandle()) { + return handlePrefix + item.getHandle(); + } else { + return "item id: " + item.getID(); + } + } + + @Override + public int perform(DSpaceObject dso) throws IOException { + int status = Curator.CURATE_UNSET; + StringBuilder results = new StringBuilder(); + String errStr = "Unknown error"; + + // do on Items only + if (dso instanceof Item) { + Item item = (Item) dso; + if (item.getHandle() != null) { + List metadataValues = itemService.getMetadata( + item, Item.ANY, Item.ANY, Item.ANY, Item.ANY); + + // no metadata? + if (metadataValues == null || metadataValues.isEmpty()) { + errStr = "Does not have any metadata"; + status = Curator.CURATE_FAIL; + } else { + // perform the validation + try { + validateDcType(item, results); + validateTitle(item, results); + validateDcLanguageIso(item, results); + validateRelations(item, results); + validateEmptyMetadata(item, metadataValues, results); + validatePredefinedNonRepeatableMetadata(item, results); + validateStrangeMetadata(item, results); + validateRightsLabels(item, results); + itemWithFilesHasLicense(item); + validateHighlyRecommendedMetadata(item, results); + validateComplexInputs(item, results); + + status = Curator.CURATE_SUCCESS; + } catch (CurateException exc) { + errStr = exc.getMessage(); + status = exc.errCode; + } + } + } else { + // no handle! + errStr = "Does not have a handle"; + status = Curator.CURATE_FAIL; + } + + // format the error if any + switch (status) { + case Curator.CURATE_SUCCESS: + break; + case CURATE_WARNING: + results.append(String.format("Warning: %s %s", errStr, addMagicString(getHandle(item)))); + break; + default: + results.append(String.format("ERROR! %s %s", errStr, addMagicString(getHandle(item)))); + break; + } + } + + report(results.toString()); + setResult(results.toString()); + return status; + } + + /** + * Add magic string for identification in reports. + * @param handle the handle to mark + * @return marked string + */ + private static String addMagicString(String handle) { + return "[[" + handle + "]]"; + } + + // + // dc type checker + // + + private void validateDcType(Item item, StringBuilder results) throws CurateException { + List dcsType = itemService.getMetadataByMetadataString(item, "dc.type"); + // no metadata? + if (dcsType == null || dcsType.isEmpty()) { + throw new CurateException("Does not have dc.type metadata", Curator.CURATE_FAIL); + } + + // check array is not null or length > 0 + for (MetadataValue dcsEntry : dcsType) { + String value = dcsEntry.getValue(); + if (value == null) { + throw new CurateException("dc.type has null value", Curator.CURATE_FAIL); + } + + String typeVal = value.trim(); + + // check if original and trimmed versions match + if (!typeVal.equals(value)) { + throw new CurateException("leading or trailing spaces", Curator.CURATE_FAIL); + } + + // check if the dc.type field is empty + if (Pattern.matches("^\\s*$", typeVal)) { + throw new CurateException("empty value", Curator.CURATE_FAIL); + } + + // check if the value is valid + if (!dcTypeValuesSet.contains(typeVal)) { + throw new CurateException("invalid type (" + typeVal + ")", Curator.CURATE_FAIL); + } + } + } + + /** + * Checks the language code (dc.language.iso) against the possible language codes + * and validates that local.language.name matches the human-readable language names. + * + * @param item the item + * @param results the results + * @throws CurateException if validation fails + */ + private void validateDcLanguageIso(Item item, StringBuilder results) throws CurateException { + List dcsLanguageIso = itemService.getMetadataByMetadataString(item, "dc.language.iso"); + + // build maps of expected and actual language names keyed by place + Map expectedLangNamesByPlace = new HashMap<>(); + Map isoCodesByPlace = new HashMap<>(); + + if (dcsLanguageIso != null && !dcsLanguageIso.isEmpty()) { + // Validate dc.language.iso codes + for (MetadataValue langCodeDC : dcsLanguageIso) { + String langCode = langCodeDC.getValue(); + if (langCode == null) { + throw new CurateException("dc.language.iso has null value", Curator.CURATE_FAIL); + } + if (IsoLangCodes.getLangForCode(langCode) == null) { + throw new CurateException( + String.format("Invalid language code - %s", langCode), + Curator.CURATE_FAIL); + } + + Integer place = langCodeDC.getPlace(); + String expectedLangName = IsoLangCodes.getLangForCode(langCode); + expectedLangNamesByPlace.put(place, expectedLangName); + isoCodesByPlace.put(place, langCode); + } + + // Validate local.language.name matches dc.language.iso + List languageNames = itemService.getMetadataByMetadataString(item, "local.language.name"); + if (languageNames == null || languageNames.size() != dcsLanguageIso.size()) { + throw new CurateException( + String.format("local.language.name count [%d] does not match dc.language.iso count [%d]", + languageNames == null ? 0 : languageNames.size(), dcsLanguageIso.size()), + Curator.CURATE_FAIL); + } + + Map actualLangNamesByPlace = new HashMap<>(); + for (MetadataValue languageName : languageNames) { + Integer place = languageName.getPlace(); + String actualLangName = languageName.getValue(); + actualLangNamesByPlace.put(place, actualLangName); + } + + // Ensure that the sets of places match between ISO codes and language names + Set expectedPlaces = expectedLangNamesByPlace.keySet(); + Set actualPlaces = actualLangNamesByPlace.keySet(); + if (!expectedPlaces.equals(actualPlaces)) { + throw new CurateException( + String.format("local.language.name places %s do not match dc.language.iso places %s", + actualPlaces, expectedPlaces), + Curator.CURATE_FAIL); + } + // Validate that each language name corresponds to its ISO code for each place + for (Integer place : expectedPlaces) { + String expectedLangName = expectedLangNamesByPlace.get(place); + String actualLangName = actualLangNamesByPlace.get(place); + if (!expectedLangName.equals(actualLangName)) { + throw new CurateException( + String.format( + "local.language.name [%s] at place [%d] does not match expected name [%s] " + + "for ISO code [%s]", + actualLangName, place, expectedLangName, isoCodesByPlace.get(place)), + Curator.CURATE_FAIL); + } + } + } + } + + // + // title checker + // + + private void validateTitle(Item item, StringBuilder results) throws CurateException { + String title = itemService.getMetadataFirstValue(item, "dc", "title", null, Item.ANY); + if (title == null) { + throw new CurateException("Item has no dc.title metadata", Curator.CURATE_FAIL); + } + if (itemTitles.containsKey(title)) { + String msg = String.format("Title [%s] duplicate in [%s]", title, itemTitles.get(title)); + throw new CurateException(msg, Curator.CURATE_FAIL); + } + itemTitles.put(title, getHandle(item)); + } + + // + // relation checker (based on assumption items are not part of multiple version histories) + // + + private void validateRelations(Item item, StringBuilder results) throws CurateException { + String handlePrefixLocal = configurationService.getProperty("handle.canonical.prefix"); + try { + String mdIsReplacedBy = "dc.relation.isreplacedby"; + String mdReplaces = "dc.relation.replaces"; + + List dcsIsReplacedBy = getNonBlankMetadata(item, mdIsReplacedBy); + List dcsReplaces = getNonBlankMetadata(item, mdReplaces); + + if (dcsIsReplacedBy.isEmpty() && dcsReplaces.isEmpty()) { + // item contains no relation metadata, nothing to check + return; + } + + // check if objects referenced by "dc.relation.isreplacedby" exist, + // and reference back to this item with "dc.relation.replaces" metadata + if (!dcsIsReplacedBy.isEmpty()) { + boolean relationsOK = + checkRelations(item, dcsIsReplacedBy, mdIsReplacedBy, mdReplaces, handlePrefixLocal); + if (!relationsOK) { + throw relationMetadataException(mdIsReplacedBy, mdReplaces); + } + } + // check if objects referenced by "dc.relation.replaces" exist, + // and reference forward to this item with "dc.relation.isreplacedby" metadata + if (!dcsReplaces.isEmpty()) { + boolean relationsOK = checkRelations(item, dcsReplaces, mdReplaces, mdIsReplacedBy, handlePrefixLocal); + if (!relationsOK) { + throw relationMetadataException(mdReplaces, mdIsReplacedBy); + } + } + + // everything is OK + results.append(String.format("Item [%s] meets relation requirements. ", getHandle(item))); + + } catch (SQLException | IOException e) { + throw new CurateException(e.getMessage(), Curator.CURATE_FAIL); + } + } + + private List getNonBlankMetadata(Item item, String metadataString) { + return itemService.getMetadataByMetadataString(item, metadataString) + .stream() + .filter(metadataValue -> !StringUtils.isBlank(metadataValue.getValue())) + .collect(Collectors.toList()); + } + + private boolean checkRelations(Item item, + List references, + String referencesFieldName, + String fieldNameInOtherDirection, + String handlePrefixLocal) throws SQLException, IOException, CurateException { + for (MetadataValue ref : references) { + Item referencedItem = getReferencedItem(ref, handlePrefixLocal); + boolean checksPass = hasReferenceBack(referencedItem, item.getHandle(), + fieldNameInOtherDirection, handlePrefixLocal) && + checkVersionHistory(item, referencedItem, referencesFieldName); + if (!checksPass) { + return false; + } + } + return true; + } + + private Item getReferencedItem(MetadataValue relatedReference, String handlePrefixLocal) + throws SQLException, IOException, CurateException { + String referencedItemHandle = getHandle(relatedReference, handlePrefixLocal); + DSpaceObject referencedObject = dereference(Curator.curationContext(), referencedItemHandle); + if (referencedObject instanceof Item) { + return (Item) referencedObject; + } else { + throw new CurateException( + String.format("contains '%s' but the referenced object [[%s]] is not an item or doesn't exist", + relatedReference.getMetadataField().toString('.'), referencedItemHandle), + Curator.CURATE_FAIL); + } + } + + private boolean hasReferenceBack(Item referencedItem, String handleBack, String fieldNameInOtherDirection, + String handlePrefixLocal) throws CurateException { + boolean ok = itemService.getMetadataByMetadataString(referencedItem, fieldNameInOtherDirection).stream() + .map(mdv -> getHandle(mdv, handlePrefixLocal)) + .anyMatch(handle -> handle != null && handle.equals(handleBack)); + if (!ok) { + throw new CurateException(String.format("the referenced item %s does not refer back via %s", + addMagicString(getHandle(referencedItem)), fieldNameInOtherDirection), Curator.CURATE_FAIL); + } + return true; + } + + private String getHandle(MetadataValue relationReference, String handlePrefixLocal) { + String handle = relationReference.getValue(); + if (StringUtils.isNotBlank(handlePrefixLocal) && handle != null && handle.startsWith(handlePrefixLocal)) { + handle = handle.substring(handlePrefixLocal.length()); + } + return handle; + } + + private boolean checkVersionHistory(Item item1, Item item2, String relation) throws SQLException, CurateException { + VersionHistory item1History = versionHistoryService.findByItem(Curator.curationContext(), item1); + if (item1History == null) { + throw new CurateException( + String.format("contains '%s' but it's not part of any version history", relation), + Curator.CURATE_FAIL + ); + } + VersionHistory item2History = versionHistoryService.findByItem(Curator.curationContext(), item2); + if (item2History == null) { + throw new CurateException( + String.format("contains '%s' but the referenced item %s is not part of any version history", + relation, addMagicString(getHandle(item2))), + Curator.CURATE_FAIL + ); + } + + if (!item1History.equals(item2History)) { + throw new CurateException( + String.format("contains '%s' but the referenced item %s is not in the same version history", + relation, addMagicString(getHandle(item2))), + Curator.CURATE_FAIL + ); + } + return true; + } + + private static CurateException relationMetadataException(String leftRel, String rightRel) { + return new CurateException( + String.format("contains '%s' but the referenced object doesn't exist or " + + "doesn't contain '%s' or doesn't point to this item", + leftRel, rightRel), + Curator.CURATE_FAIL + ); + } + + private void validateEmptyMetadata(Item item, List metadataValues, StringBuilder results) + throws CurateException { + for (MetadataValue dc : metadataValues) { + if (dc.getValue() == null) { + throw new CurateException( + String.format("value [%s.%s.%s] is null", dc.getMetadataField().getMetadataSchema().getName(), + dc.getMetadataField().getElement(), dc.getMetadataField().getQualifier()), + Curator.CURATE_FAIL); + } + if (dc.getValue().trim().length() == 0) { + throw new CurateException( + String.format("value [%s.%s.%s] is empty", dc.getMetadataField().getMetadataSchema().getName(), + dc.getMetadataField().getElement(), dc.getMetadataField().getQualifier()), + Curator.CURATE_FAIL); + } + } + } + + private void validatePredefinedNonRepeatableMetadata(Item item, StringBuilder results) throws CurateException { + for (String noDuplicate : nonRepeatableMetadata) { + List vals = itemService.getMetadataByMetadataString(item, noDuplicate); + if (null != vals && vals.size() > 1) { + throw new CurateException( + String.format("value [%s] is present multiple times", noDuplicate), + Curator.CURATE_FAIL); + } + } + } + + private void validateRightsLabels(Item item, StringBuilder results) throws CurateException { + List dcvs = itemService.getMetadata(item, "dc", "rights", "label", Item.ANY); + try { + // Only check if item has files when we have an active session + // Skip this check if we can't access bundles (lazy loading issue) + if (null != item.getHandle() && dcvs != null && !dcvs.isEmpty()) { + if (!itemService.hasUploadedFiles(item, "ORIGINAL")) { + StringBuilder labels = new StringBuilder(); + for (MetadataValue label : dcvs) { + labels.append(label.getValue()).append(" "); + } + throw new CurateException( + String.format("has labels [%s] but no files", labels.toString()), + Curator.CURATE_FAIL); + } + } + } catch (SQLException e) { + throw new CurateException( + String.format("has internal problems [%s]", e.getMessage()), + Curator.CURATE_FAIL); + } + } + + private void validateHighlyRecommendedMetadata(Item item, StringBuilder results) throws CurateException { + for (String md : highlyRecommended) { + List vals = itemService.getMetadataByMetadataString(item, md); + if (null == vals || vals.isEmpty()) { + throw new CurateException( + String.format("does not contain any [%s] values", md), + CURATE_WARNING); + } + } + } + + private void validateStrangeMetadata(Item item, StringBuilder results) throws CurateException { + for (String md : strangeMetadata) { + List vals = itemService.getMetadataByMetadataString(item, md); + if (null != vals && !vals.isEmpty()) { + throw new CurateException( + String.format("contains suspicious [%s] metadata", md), + Curator.CURATE_FAIL); + } + } + } + + private void validateComplexInputs(Item item, StringBuilder results) throws CurateException { + for (Map.Entry entry : complexInputs.entrySet()) { + for (MetadataValue dval : itemService.getMetadataByMetadataString(item, entry.getKey())) { + String val = dval.getValue(); + if (val.split(DCInput.ComplexDefinitions.getSeparator(), -1).length != entry.getValue()) { + throw new CurateException( + String.format( + "%s is a component with %s values but is not stored as such. [%s]", + entry.getKey(), entry.getValue(), val), + Curator.CURATE_FAIL); + } + } + } + } + + private void itemWithFilesHasLicense(Item item) throws CurateException { + try { + boolean fail = false; + StringBuilder sb = new StringBuilder(); + try { + if (itemService.hasUploadedFiles(item, "ORIGINAL")) { + for (String mdString : rightsMdStrings) { + final List vals = itemService.getMetadataByMetadataString(item, mdString); + if (vals == null || vals.isEmpty()) { + fail = true; + sb.append(mdString).append(", "); + } + } + } + } catch (org.hibernate.LazyInitializationException e) { + // Item is detached from session, skip file check + // This can happen when processing large batches + log.debug("Skipping file check for item {} due to detached session", item.getHandle()); + } + if (fail) { + throw new CurateException("There are bitstreams but incomplete rights metadata. Missing: " + + sb.toString(), Curator.CURATE_FAIL); + } + } catch (SQLException throwables) { + throw new CurateException(throwables.getMessage(), Curator.CURATE_ERROR); + } + } + + /** + * Curate exception. + */ + static class CurateException extends Exception { + int errCode; + + public CurateException(String message, int errCode) { + super(message); + this.errCode = errCode; + } + } +} diff --git a/dspace-api/src/test/data/dspaceFolder/config/modules/curate.cfg b/dspace-api/src/test/data/dspaceFolder/config/modules/curate.cfg index 22b44f319a26..b7d0aec48f70 100644 --- a/dspace-api/src/test/data/dspaceFolder/config/modules/curate.cfg +++ b/dspace-api/src/test/data/dspaceFolder/config/modules/curate.cfg @@ -15,6 +15,7 @@ plugin.named.org.dspace.curate.CurationTask = org.dspace.ctask.general.RequiredM #plugin.named.org.dspace.curate.CurationTask = org.dspace.ctask.general.MicrosoftTranslator = translate plugin.named.org.dspace.curate.CurationTask = org.dspace.ctask.general.MetadataValueLinkChecker = checklinks plugin.named.org.dspace.curate.CurationTask = org.dspace.ctask.general.ItemHandleChecker = checkhandles +plugin.named.org.dspace.curate.CurationTask = org.dspace.ctask.general.ItemMetadataQAChecker = metadataqa # add new tasks here (or in additional config files) # Testing tasks diff --git a/dspace-api/src/test/java/org/dspace/curate/ItemMetadataQACheckerIT.java b/dspace-api/src/test/java/org/dspace/curate/ItemMetadataQACheckerIT.java new file mode 100644 index 000000000000..315a04a68b11 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/curate/ItemMetadataQACheckerIT.java @@ -0,0 +1,458 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.curate; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.Date; + +import org.dspace.AbstractIntegrationTestWithDatabase; +import org.dspace.authorize.AuthorizeException; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.content.Collection; +import org.dspace.content.Community; +import org.dspace.content.Item; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.CollectionService; +import org.dspace.content.service.CommunityService; +import org.dspace.content.service.ItemService; +import org.dspace.core.factory.CoreServiceFactory; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; +import org.dspace.versioning.VersionHistory; +import org.dspace.versioning.factory.VersionServiceFactory; +import org.dspace.versioning.service.VersionHistoryService; +import org.dspace.versioning.service.VersioningService; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Test for ItemMetadataQAChecker curation task. + * + * @author LINDAT/CLARIN + */ +public class ItemMetadataQACheckerIT extends AbstractIntegrationTestWithDatabase { + private static final String TASK_NAME = "metadataqa"; + + protected CommunityService communityService = ContentServiceFactory.getInstance().getCommunityService(); + protected CollectionService collectionService = ContentServiceFactory.getInstance().getCollectionService(); + protected ItemService itemService = ContentServiceFactory.getInstance().getItemService(); + protected VersionHistoryService versionHistoryService = + VersionServiceFactory.getInstance().getVersionHistoryService(); + protected VersioningService versioningService = VersionServiceFactory.getInstance().getVersionService(); + protected ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); + + Community parentCommunity; + Collection collection; + Item validItem; + Item itemWithoutDcType; + Item itemWithInvalidDcType; + Item itemWithInvalidLanguage; + Item itemWithIncorrectLanguageName; + Item itemWithTwoAvailableDates; + Item itemWithTwoAvailableDatesAndLang; + Item itemVersion1; + Item itemVersion2; + Item itemVersion3; + Item itemVersion4; + Item itemVersion5; + private String handlePrefix; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + CoreServiceFactory.getInstance().getPluginService().clearNamedPluginClasses(); + try { + context.turnOffAuthorisationSystem(); + + // Create a parent community + this.parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Test Community") + .build(); + + // Create a collection + this.collection = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Test Collection") + .build(); + + // Create a valid item with all required metadata + validItem = ItemBuilder.createItem(context, collection) + .withTitle("Valid Test Item") + .withMetadata("dc", "type", null, "corpus") + .withMetadata("dc", "language", "iso", "eng") + .withMetadata("local", "language", "name", "English") + .withMetadata("dc", "subject", null, "test subject") + .withMetadata("local", "branding", null, "Test Community") + .build(); + + // Create an item without dc.type + itemWithoutDcType = ItemBuilder.createItem(context, collection) + .withTitle("Item Without Type") + .build(); + + // Create an item with invalid dc.type + itemWithInvalidDcType = ItemBuilder.createItem(context, collection) + .withTitle("Item With Invalid Type") + .withMetadata("dc", "type", null, "invalidType") + .build(); + + // Create an item with invalid language code + itemWithInvalidLanguage = ItemBuilder.createItem(context, collection) + .withTitle("Item With Invalid Language") + .withMetadata("dc", "type", null, "corpus") + .withMetadata("dc", "language", "iso", "xyz") + .build(); + + // Create an item with incorrect local.language.name - deliberately set wrong name + // Note: We need to create it without triggering automatic language name addition + itemWithIncorrectLanguageName = ItemBuilder.createItem(context, collection) + .withTitle("Item With Incorrect Language Name") + .withMetadata("dc", "type", null, "corpus") + .withMetadata("dc", "subject", null, "test subject") + .withMetadata("local", "branding", null, "Test Community") + .build(); + // Manually add dc.language.iso and wrong local.language.name after creation + itemService.addMetadata(context, itemWithIncorrectLanguageName, "dc", "language", "iso", null, "eng"); + itemService.addMetadata(context, itemWithIncorrectLanguageName, "local", "language", "name", null, + "WrongLanguageName"); + itemService.update(context, itemWithIncorrectLanguageName); + + itemWithTwoAvailableDates = ItemBuilder.createItem(context, collection) + .withTitle("Item With Two Available Dates") + .withMetadata("dc", "type", null, "corpus") + .withMetadata("dc", "date", "available", "2020-01-01") + .withMetadata("dc", "date", "available", "2021-01-01") + .build(); + + itemWithTwoAvailableDatesAndLang = ItemBuilder.createItem(context, collection) + .withTitle("Item With Two Available Dates") + .withMetadata("dc", "type", null, "corpus") + .withMetadata("dc", "date", "available", "2020-01-01") + .build(); + + itemService.addMetadata(context, itemWithTwoAvailableDatesAndLang,"dc", "date", + "available", "en_US", "2021-01-01"); + + itemVersion1 = ItemBuilder.createItem(context, collection) + .withTitle("Item Version 1") + .withMetadata("dc", "type", null, "corpus") + .withMetadata("dc", "subject", null, "test subject") + .withMetadata("local", "branding", null, "Test Community") + .build(); + + itemVersion2 = ItemBuilder.createItem(context, collection) + .withTitle("Item Version 2") + .withMetadata("dc", "type", null, "corpus") + .withMetadata("dc", "subject", null, "test subject") + .withMetadata("local", "branding", null, "Test Community") + .build(); + + itemVersion3 = ItemBuilder.createItem(context, collection) + .withTitle("Item Version 3") + .withMetadata("dc", "type", null, "corpus") + .withMetadata("dc", "subject", null, "test subject") + .withMetadata("local", "branding", null, "Test Community") + .build(); + + itemVersion4 = ItemBuilder.createItem(context, collection) + .withTitle("Item Version 4") + .withMetadata("dc", "type", null, "corpus") + .withMetadata("dc", "subject", null, "test subject") + .withMetadata("local", "branding", null, "Test Community") + .build(); + + itemVersion5 = ItemBuilder.createItem(context, collection) + .withTitle("Item Version 5") + .withMetadata("dc", "type", null, "corpus") + .withMetadata("dc", "subject", null, "test subject") + .withMetadata("local", "branding", null, "Test Community") + .build(); + + String ref1 = itemService.getMetadataFirstValue(itemVersion1, "dc", "identifier", "uri", Item.ANY); + String ref2 = itemService.getMetadataFirstValue(itemVersion2, "dc", "identifier", "uri", Item.ANY); + + itemService.addMetadata(context, itemVersion1, "dc", "relation", "isreplacedby", null, ref2); + itemService.addMetadata(context, itemVersion2, "dc", "relation", "replaces", null, ref1); + itemService.addMetadata(context, itemVersion3, "dc", "relation", "replaces", null, ref2); + itemService.update(context, itemVersion1); + itemService.update(context, itemVersion2); + itemService.update(context, itemVersion3); + + VersionHistory versionHistory = versionHistoryService.create(context); + versioningService.createNewVersion(context, versionHistory, itemVersion1, "Version 1", new Date(), 1); + versioningService.createNewVersion(context, versionHistory, itemVersion2, "Version 2", new Date(), 2); + versioningService.createNewVersion(context, versionHistory, itemVersion3, "Version 3", new Date(), 3); + + context.restoreAuthSystemState(); + handlePrefix = configurationService.getProperty("handle.canonical.prefix"); + + } catch (Exception ex) { + fail("Error in init: " + ex.getMessage()); + } + } + + @Test + public void testItemWithTwoAvailableDates() throws IOException { + Curator curator = new Curator(); + curator.addTask(TASK_NAME); + context.setCurrentUser(admin); + + // Run curator task for item with two dc.date.available - should fail + curator.curate(context, itemWithTwoAvailableDates.getHandle()); + int status = curator.getStatus(TASK_NAME); + assertEquals("Curation should fail for item with two dc.date.available", Curator.CURATE_FAIL, status); + String result = curator.getResult(TASK_NAME); + assertTrue("Result should mention multiple dc.date.available", result.contains("dc.date.available")); + } + + @Test + public void testItemWithTwoAvailableDatesAndLang() throws IOException { + Curator curator = new Curator(); + curator.addTask(TASK_NAME); + context.setCurrentUser(admin); + + // Run curator task for item with two dc.date.available with language - should fail + curator.curate(context, itemWithTwoAvailableDatesAndLang.getHandle()); + int status = curator.getStatus(TASK_NAME); + assertEquals("Curation should fail for item with two dc.date.available with language", + Curator.CURATE_FAIL, status); + String result = curator.getResult(TASK_NAME); + assertTrue("Result should mention multiple dc.date.available", result.contains("dc.date.available")); + } + + @Test + public void testValidItem() throws IOException { + Curator curator = new Curator(); + curator.addTask(TASK_NAME); + context.setCurrentUser(admin); + + // Run curator task for valid item - should succeed + curator.curate(context, validItem.getHandle()); + int status = curator.getStatus(TASK_NAME); + assertEquals("Curation should succeed for valid item", Curator.CURATE_SUCCESS, status); + } + + @Test + public void testItemWithoutDcType() throws IOException { + Curator curator = new Curator(); + curator.addTask(TASK_NAME); + context.setCurrentUser(admin); + + // Run curator task for item without dc.type - should fail + curator.curate(context, itemWithoutDcType.getHandle()); + int status = curator.getStatus(TASK_NAME); + assertEquals("Curation should fail for item without dc.type", Curator.CURATE_FAIL, status); + String result = curator.getResult(TASK_NAME); + assertTrue("Result should mention dc.type metadata", result.contains("dc.type")); + } + + @Test + public void testItemWithInvalidDcType() throws IOException { + Curator curator = new Curator(); + curator.addTask(TASK_NAME); + context.setCurrentUser(admin); + + // Run curator task for item with invalid dc.type - should fail + curator.curate(context, itemWithInvalidDcType.getHandle()); + int status = curator.getStatus(TASK_NAME); + assertEquals("Curation should fail for item with invalid dc.type", Curator.CURATE_FAIL, status); + String result = curator.getResult(TASK_NAME); + assertTrue("Result should mention invalid type", result.contains("invalid type")); + } + + @Test + public void testItemWithInvalidLanguageCode() throws IOException { + Curator curator = new Curator(); + curator.addTask(TASK_NAME); + context.setCurrentUser(admin); + + // Run curator task for item with invalid language code - should fail + curator.curate(context, itemWithInvalidLanguage.getHandle()); + int status = curator.getStatus(TASK_NAME); + assertEquals("Curation should fail for item with invalid language code", Curator.CURATE_FAIL, status); + String result = curator.getResult(TASK_NAME); + assertTrue("Result should mention invalid language code", result.contains("Invalid language code")); + } + + @Test + public void testItemWithIncorrectLanguageName() throws IOException { + Curator curator = new Curator(); + curator.addTask(TASK_NAME); + context.setCurrentUser(admin); + + // Run curator task for item with incorrect local.language.name - should fail + curator.curate(context, itemWithIncorrectLanguageName.getHandle()); + int status = curator.getStatus(TASK_NAME); + String result = curator.getResult(TASK_NAME); + assertEquals("Curation should fail for item with incorrect local.language.name", Curator.CURATE_FAIL, status); + assertTrue("Result should mention local.language.name mismatch, but was: " + result, + result.contains("local.language.name") && result.contains("does not match")); + } + + @Test + public void testItemVersion1() throws IOException { + testItemWithCorrectRelationship(itemVersion1, "meets relation requirements"); + } + + @Test + public void testItemVersion2() throws IOException { + testItemWithCorrectRelationship(itemVersion2, "meets relation requirements"); + } + + @Test + public void testItemWithBadRelationship1() throws IOException, SQLException, AuthorizeException { + // itemVersion2 has 'dc.relation.isreplacedby that points to itemVersion4 + // but itemVersion4 doesn't contain 'dc.relation.replaces' metadata + String ref4 = itemService.getMetadataFirstValue(itemVersion4, "dc", "identifier", "uri", Item.ANY); + + context.turnOffAuthorisationSystem(); + itemService.addMetadata(context, itemVersion2, "dc", "relation", "isreplacedby", null, ref4); + itemService.update(context, itemVersion2); + context.restoreAuthSystemState(); + + testItemWithRelationError( + itemVersion2, + "the referenced item [[%s]] does not refer back via %s", + ref4, + "dc.relation.replaces"); + } + + @Test + public void testItemWithBadRelationship2() throws IOException { + String ref2 = itemService.getMetadataFirstValue(itemVersion2, "dc", "identifier", "uri", Item.ANY); + // itemVersion3 has 'dc.relation.replaces' that points back to itemVersion2 + // but itemVersion2 doesn't have 'dc.relation.isreplacedby' that points forward to itemVersion3 + testItemWithRelationError( + itemVersion3, + "the referenced item [[%s]] does not refer back via %s", + ref2, + "dc.relation.isreplacedby"); + } + @Test + public void testItemWithBadRelationship3() throws IOException, SQLException, AuthorizeException { + + context.turnOffAuthorisationSystem(); + String ref = "https://example.org/this-doesnt-resolve"; + itemService.addMetadata(context, itemVersion5, "dc", "relation", "replaces", null, ref); + itemService.update(context, itemVersion5); + context.restoreAuthSystemState(); + + testItemWithRelationError( + itemVersion5, + "contains '%s' but the referenced object [[%s]] is not an item or doesn't exist", + "dc.relation.replaces", + ref); + } + + @Test + public void testItemWithMissingVersionHistory() throws SQLException, IOException, AuthorizeException { + String ref2 = itemService.getMetadataFirstValue(itemVersion2, "dc", "identifier", "uri", Item.ANY); + String ref4 = itemService.getMetadataFirstValue(itemVersion4, "dc", "identifier", "uri", Item.ANY); + + context.turnOffAuthorisationSystem(); + itemService.addMetadata(context, itemVersion2, "dc", "relation", "isreplacedby", null, ref4); + itemService.addMetadata(context, itemVersion4, "dc", "relation", "replaces", null, ref2); + itemService.update(context, itemVersion2); + itemService.update(context, itemVersion4); + context.restoreAuthSystemState(); + + testItemWithRelationError(itemVersion4, + "contains '%s' but it's not part of any version history", "dc.relation.replaces"); + } + + @Test + public void testItemWithMissingVersionHistoryForReferencedItem() + throws SQLException, IOException, AuthorizeException { + String ref2 = itemService.getMetadataFirstValue(itemVersion2, "dc", "identifier", "uri", Item.ANY); + String ref4 = itemService.getMetadataFirstValue(itemVersion4, "dc", "identifier", "uri", Item.ANY); + + context.turnOffAuthorisationSystem(); + itemService.addMetadata(context, itemVersion2, "dc", "relation", "isreplacedby", null, ref4); + itemService.addMetadata(context, itemVersion4, "dc", "relation", "replaces", null, ref2); + itemService.update(context, itemVersion2); + itemService.update(context, itemVersion4); + context.restoreAuthSystemState(); + + testItemWithRelationError(itemVersion2, + "contains '%s' but the referenced item [[%s]] is not part of any version history", + "dc.relation.isreplacedby", ref4); + } + + @Test + public void testItemWithNotMatchingVersionHistory() throws SQLException, IOException, AuthorizeException { + String ref2 = itemService.getMetadataFirstValue(itemVersion2, "dc", "identifier", "uri", Item.ANY); + String ref4 = itemService.getMetadataFirstValue(itemVersion4, "dc", "identifier", "uri", Item.ANY); + + context.turnOffAuthorisationSystem(); + itemService.addMetadata(context, itemVersion2,"dc", "relation", "isreplacedby", null, ref4); + itemService.addMetadata(context, itemVersion4,"dc", "relation", "replaces", null, ref2); + itemService.update(context, itemVersion2); + itemService.update(context, itemVersion4); + context.restoreAuthSystemState(); + + VersionHistory versionHistory = versionHistoryService.create(context); + versioningService.createNewVersion(context, versionHistory, itemVersion4, + "Another Version History - Version 1", new Date(), 1); + + testItemWithRelationError(itemVersion4, + "contains '%s' but the referenced item [[%s]] is not in the same version history", + "dc.relation.replaces", ref2); + } + + @Test + public void testItemWithNoRelationMetadata() throws SQLException, IOException { + testItemWithCorrectRelationship(itemVersion4, null); + } + + private void testItemWithCorrectRelationship(Item item, String successMessage) throws IOException { + Curator curator = runCuratorForItem(item); + + int status = curator.getStatus(TASK_NAME); + String result = curator.getResult(TASK_NAME); + assertEquals("Curation should succeed for valid item with relation", Curator.CURATE_SUCCESS, status); + if (successMessage == null) { + assertTrue("Result must be empty, but was " + result, result.isEmpty()); + } else { + assertTrue("Result must contain success message, but was " + result, + result.contains(successMessage) && result.contains(item.getHandle())); + } + } + + private void testItemWithRelationError(Item item, String errorMessage, Object... args) throws IOException { + Curator curator = runCuratorForItem(item); + + int status = curator.getStatus(TASK_NAME); + String result = curator.getResult(TASK_NAME); + assertEquals("Curation should fail for incorrect relationship", Curator.CURATE_FAIL, status); + String failMessage = String.format(errorMessage, args); + assertTrue(String.format("Result: %s\n must contain fail message \n %s ", result, failMessage), + result.contains(failMessage) + ); + } + + private Curator runCuratorForItem(Item item) throws IOException { + Curator curator = new Curator(); + curator.addTask(TASK_NAME); + context.setCurrentUser(admin); + curator.curate(context, item.getHandle()); + return curator; + } + + @After + public void destroy() throws Exception { + super.destroy(); + } +} diff --git a/dspace/config/modules/curate.cfg b/dspace/config/modules/curate.cfg index d36578fa943d..62e6e3644d8f 100644 --- a/dspace/config/modules/curate.cfg +++ b/dspace/config/modules/curate.cfg @@ -16,6 +16,7 @@ plugin.named.org.dspace.curate.CurationTask = org.dspace.ctask.general.RequiredM plugin.named.org.dspace.curate.CurationTask = org.dspace.ctask.general.MetadataValueLinkChecker = checklinks plugin.named.org.dspace.curate.CurationTask = org.dspace.ctask.general.ItemHandleChecker = checkhandles plugin.named.org.dspace.curate.CurationTask = org.dspace.ctask.general.RegisterDOI = registerdoi +plugin.named.org.dspace.curate.CurationTask = org.dspace.ctask.general.ItemMetadataQAChecker = metadataqa #plugin.named.org.dspace.curate.CurationTask = org.dspace.ctask.general.CitationPage = citationpage # add new tasks here (or in additional config files) From d0c04cbdbbfe151cf8d4aad3bb15c1ae717ba0e1 Mon Sep 17 00:00:00 2001 From: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> Date: Tue, 26 May 2026 15:22:29 +0200 Subject: [PATCH 25/41] UFAL/Separate CLARIN license payload from sections.license (#1319) * fix(submission): separate CLARIN license payload from sections.license * fix(clarin-license): rename step path to /select, rename DataClarinLicense to ClarinDataLicense, and clean up comments * fix(clarin-license): align DTO name with Rest suffix convention * fix(submission): align CLARIN section DTO naming and apply Copilot review fixes * Align CLARIN license patch semantics and tighten section path handling * Fix checkstyle issue * Stabilize unknown CLARIN license metadata assertion * Added doc and checked null value * Harden CLARIN license patch handling and logging --- .../rest/model/step/ClarinDataLicense.java | 61 +++++ .../WorkspaceItemRestRepository.java | 61 +---- .../step/ClarinLicenseResourceStep.java | 160 +++++++++--- .../step/ClarinLicenseSubmissionUtils.java | 122 +++++++++ .../ClarinWorkspaceItemRestRepositoryIT.java | 235 ++++++++++++++++++ 5 files changed, 554 insertions(+), 85 deletions(-) create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/model/step/ClarinDataLicense.java create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/step/ClarinLicenseSubmissionUtils.java diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/step/ClarinDataLicense.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/step/ClarinDataLicense.java new file mode 100644 index 000000000000..a6bdfa803a96 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/step/ClarinDataLicense.java @@ -0,0 +1,61 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.model.step; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty.Access; + +/** + * Java Bean to expose the CLARIN license section during in progress submission. + * + * @author Milan Majchrak (milan.majchrak at dataquest.sk) + */ +public class ClarinDataLicense implements SectionData { + + private String name; + + @JsonProperty(access = Access.READ_ONLY) + private String definition; + + @JsonProperty(access = Access.READ_ONLY) + private String label; + + private boolean granted = false; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDefinition() { + return definition; + } + + public void setDefinition(String definition) { + this.definition = definition; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public boolean isGranted() { + return granted; + } + + public void setGranted(boolean granted) { + this.granted = granted; + } +} \ No newline at end of file diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java index a229d8b8c8c7..b4f29213af85 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java @@ -44,6 +44,7 @@ import org.dspace.app.rest.repository.handler.service.UriListHandlerService; import org.dspace.app.rest.submit.SubmissionService; import org.dspace.app.rest.submit.UploadableStep; +import org.dspace.app.rest.submit.step.ClarinLicenseSubmissionUtils; import org.dspace.app.rest.utils.BigMultipartFile; import org.dspace.app.rest.utils.Utils; import org.dspace.app.util.SubmissionConfig; @@ -51,14 +52,11 @@ import org.dspace.app.util.SubmissionStepConfig; import org.dspace.authorize.AuthorizeException; import org.dspace.authorize.service.AuthorizeService; -import org.dspace.content.Bitstream; -import org.dspace.content.Bundle; import org.dspace.content.Collection; import org.dspace.content.Item; import org.dspace.content.LicenseUtils; import org.dspace.content.MetadataValue; import org.dspace.content.WorkspaceItem; -import org.dspace.content.clarin.ClarinLicense; import org.dspace.content.service.BitstreamFormatService; import org.dspace.content.service.BitstreamService; import org.dspace.content.service.CollectionService; @@ -532,12 +530,13 @@ private void maintainLicensesForItem(Context context, WorkspaceItem source, Oper // Get item Item item = source.getItem(); if (Objects.isNull(item)) { - // add log + log.warn("Cannot maintain CLARIN licenses: workspace item {} has no underlying item.", source.getID()); return; } // Get value from operation if (!(op instanceof ReplaceOperation)) { - // add log + log.warn("Ignoring non-replace operation '{}' on license patch path for workspace item {}.", + op.getOp(), source.getID()); return; } @@ -555,53 +554,13 @@ private void maintainLicensesForItem(Context context, WorkspaceItem source, Oper clarinLicenseName = jsonNodeValue.asText(); } - // Get clarin license by definition - ClarinLicense clarinLicense = clarinLicenseService.findByName(context, clarinLicenseName); - if (StringUtils.isNotBlank(clarinLicenseName) && Objects.isNull(clarinLicense)) { - throw new ClarinLicenseNotFoundException("Cannot patch workspace item with id: " + source.getID() + "," + - " because the clarin license with name: " + clarinLicenseName + " isn't supported in" + - " the CLARIN/DSpace"); - } - - // Clear the license metadata from the item - clarinLicenseService.clearLicenseMetadataFromItem(context, item); - - // Detach the clarin licenses from the uploaded bitstreams - List bundles = item.getBundles(Constants.CONTENT_BUNDLE_NAME); - for (Bundle bundle : bundles) { - List bitstreamList = bundle.getBitstreams(); - for (Bitstream bitstream : bitstreamList) { - // in case bitstream ID exists in license table for some reason .. just remove it - this.clarinLicenseResourceMappingService.detachLicenses(context, bitstream); - } - } - - // Save changes to database - itemService.update(context, item); - - if (Objects.isNull(clarinLicense)) { - log.info("The clarin license is null so all item metadata for license was cleared and the" + - "licenses was detached."); - return; - } - - // If the clarin license is not null that means some clarin license was updated and accepted - // Attach the new clarin license to every bitstream and add clarin license values to the item metadata. - - // update item metadata with license data - clarinLicenseService.addLicenseMetadataToItem(context, clarinLicense, item); - - // Attach the clarin license to the bitstreams - for (Bundle bundle : bundles) { - List bitstreamList = bundle.getBitstreams(); - for (Bitstream bitstream : bitstreamList) { - // in case bitstream ID exists in license table for some reason .. just remove it - this.clarinLicenseResourceMappingService.attachLicense(context, clarinLicense, bitstream); - } + // Delegate to the shared helper so the legacy `/license` path and the + // section path `/sections/clarin-license/select` apply the same logic. + try { + ClarinLicenseSubmissionUtils.applyLicense(context, item, clarinLicenseName); + } catch (ClarinLicenseNotFoundException ex) { + throw new UnprocessableEntityException(ex.getMessage(), ex); } - - // Save changes to database - itemService.update(context, item); } private void grantDistributionLicense(Context context, WorkspaceItem source, Operation op) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/step/ClarinLicenseResourceStep.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/step/ClarinLicenseResourceStep.java index 84babbbd7b27..b3e315af445e 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/step/ClarinLicenseResourceStep.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/step/ClarinLicenseResourceStep.java @@ -7,53 +7,71 @@ */ package org.dspace.app.rest.submit.step; +import java.util.List; import javax.servlet.http.HttpServletRequest; -import org.atteo.evo.inflector.English; +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.dspace.app.rest.exception.ClarinLicenseNotFoundException; import org.dspace.app.rest.exception.UnprocessableEntityException; -import org.dspace.app.rest.model.BitstreamRest; +import org.dspace.app.rest.model.patch.JsonValueEvaluator; import org.dspace.app.rest.model.patch.Operation; -import org.dspace.app.rest.model.step.DataLicense; +import org.dspace.app.rest.model.patch.ReplaceOperation; +import org.dspace.app.rest.model.step.ClarinDataLicense; import org.dspace.app.rest.submit.AbstractProcessingStep; import org.dspace.app.rest.submit.SubmissionService; -import org.dspace.app.rest.submit.factory.PatchOperationFactory; -import org.dspace.app.rest.submit.factory.impl.PatchOperation; import org.dspace.app.util.SubmissionStepConfig; -import org.dspace.content.Bitstream; import org.dspace.content.InProgressSubmission; -import org.dspace.core.Constants; +import org.dspace.content.Item; +import org.dspace.content.MetadataValue; import org.dspace.core.Context; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Clarin License Resource License step for DSpace Spring Rest. This Step will show license selector - * where the user could choose license for the bitstream. + * Submission step exposing the CLARIN resource license selected for the + * in-progress submission. Data is sourced from the item's {@code dc.rights*} + * metadata; the selection is updated via a section-scoped patch + * {@code /sections/clarin-license/select}. * * @author Milan Majchrak (milan.majchrak at dataquest.sk) - * - * This class is inspired by the class LicenseStep created by - * @author Luigi Andrea Pascarelli (luigiandrea.pascarelli at 4science.it) - * */ public class ClarinLicenseResourceStep extends AbstractProcessingStep { - private static final String DCTERMS_RIGHTSDATE = "dcterms.accessRights"; + private static final Logger log = LoggerFactory.getLogger(ClarinLicenseResourceStep.class); + + /** + * Sub-path of the section patch used to select a CLARIN license by name, + * e.g. {@code /sections/clarin-license/select}. + */ + private static final String LICENSE_SELECT_OPERATION_ENTRY = "select"; @Override - public DataLicense getData(SubmissionService submissionService, InProgressSubmission obj, - SubmissionStepConfig config) - throws Exception { - DataLicense result = new DataLicense(); - Bitstream bitstream = bitstreamService - .getBitstreamByName(obj.getItem(), Constants.LICENSE_BUNDLE_NAME, Constants.LICENSE_BITSTREAM_NAME); - if (bitstream != null) { - String acceptanceDate = bitstreamService.getMetadata(bitstream, DCTERMS_RIGHTSDATE); - result.setAcceptanceDate(acceptanceDate); - result.setUrl( - configurationService.getProperty("dspace.server.url") - + "/api/" + BitstreamRest.CATEGORY + "/" + English - .plural(BitstreamRest.NAME) + "/" + bitstream.getID() + "/content"); - result.setGranted(true); + public ClarinDataLicense getData(SubmissionService submissionService, InProgressSubmission obj, + SubmissionStepConfig config) { + ClarinDataLicense result = new ClarinDataLicense(); + Item item = obj.getItem(); + if (item == null) { + return result; + } + + List name = itemService.getMetadataByMetadataString(item, "dc.rights"); + List uri = itemService.getMetadataByMetadataString(item, "dc.rights.uri"); + List label = itemService.getMetadataByMetadataString(item, "dc.rights.label"); + + if (CollectionUtils.isNotEmpty(name)) { + result.setName(name.get(0).getValue()); + } + if (CollectionUtils.isNotEmpty(uri)) { + result.setDefinition(uri.get(0).getValue()); + } + if (CollectionUtils.isNotEmpty(label)) { + result.setLabel(label.get(0).getValue()); } + result.setGranted(CollectionUtils.isNotEmpty(name) + && CollectionUtils.isNotEmpty(uri) + && CollectionUtils.isNotEmpty(label)); return result; } @@ -61,14 +79,88 @@ public DataLicense getData(SubmissionService submissionService, InProgressSubmis public void doPatchProcessing(Context context, HttpServletRequest currentRequest, InProgressSubmission source, Operation op, SubmissionStepConfig stepConf) throws Exception { - if (op.getPath().endsWith(LICENSE_STEP_OPERATION_ENTRY)) { + String path = op.getPath(); - PatchOperation patchOperation = new PatchOperationFactory() - .instanceOf(LICENSE_STEP_OPERATION_ENTRY, op.getOp()); - patchOperation.perform(context, currentRequest, source, op); + if (path.endsWith("/" + LICENSE_SELECT_OPERATION_ENTRY)) { + if (!(op instanceof ReplaceOperation)) { + throw new UnprocessableEntityException( + "The operation '" + op.getOp() + "' is not supported for path " + path); + } + String licenseName = extractLicenseName(op); + // Section endpoint: a missing or blank license name is treated as a + // client error (422). The legacy `/license` path in + // WorkspaceItemRestRepository intentionally treats a blank value as + // "clear the current license" for backwards compatibility. + if (StringUtils.isBlank(licenseName)) { + throw new UnprocessableEntityException( + "The patch value for path " + path + " must contain a non-empty license name."); + } + try { + ClarinLicenseSubmissionUtils.applyLicense(context, source.getItem(), licenseName); + } catch (ClarinLicenseNotFoundException ex) { + // Surface invalid client input as 422 instead of leaking as 500. + throw new UnprocessableEntityException(ex.getMessage(), ex); + } + return; + } + + if (path.endsWith(LICENSE_STEP_OPERATION_ENTRY)) { + // `granted` patches are a no-op on this section; kept for older clients. + log.info("Ignoring legacy '{}/granted' patch on the CLARIN license section.", stepConf.getId()); + return; + } - } else { - throw new UnprocessableEntityException("The path " + op.getPath() + " cannot be patched"); + throw new UnprocessableEntityException("The path " + path + " cannot be patched"); + } + + /** + * Extract the CLARIN license name from a JSON Patch {@link Operation}. + *

+ * The submission API receives section updates as JSON Patch operations + * (see {@code /sections/clarin-license/select}). The {@code value} field + * of such an operation is not strongly typed: depending on the request + * shape and how the JSON Patch payload was parsed upstream, it can arrive + * as: + *

    + *
  • a plain {@link String}, e.g. {@code "value": "CC-BY"};
    • + *
  • a {@link JsonValueEvaluator} wrapping a {@link JsonNode}, when the + * payload is sent as a JSON object such as + * {@code "value": { "value": "CC-BY" }} or as a bare textual node;
    • + *
  • {@code null} when the client omitted the value entirely.
    • + *
+ * This helper normalizes those cases into a single {@code String} license + * name (or {@code null} if no usable value is present), so the rest of the + * step can call {@link ClarinLicenseSubmissionUtils#applyLicense} with a + * simple value and treat missing input as a client error. + * + * @param op the JSON Patch operation targeting the license {@code select} path + * @return the license name extracted from the operation value, or {@code null} + * if the operation has no usable value (missing, null, or of an + * unsupported type) + */ + private String extractLicenseName(Operation op) { + Object value = op.getValue(); + if (value == null) { + return null; + } + if (value instanceof String) { + return (String) value; + } + if (value instanceof JsonValueEvaluator) { + JsonNode valueNode = ((JsonValueEvaluator) value).getValueNode(); + if (valueNode == null) { + return null; + } + JsonNode inner = valueNode.get("value"); + if (inner != null) { + return inner.asText(); + } + if (valueNode.isTextual()) { + return valueNode.asText(); + } } + log.warn("Unsupported Operation value type for license name extraction: {}", + value.getClass().getName()); + return null; } } diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/step/ClarinLicenseSubmissionUtils.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/step/ClarinLicenseSubmissionUtils.java new file mode 100644 index 000000000000..ccc6e886408f --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/step/ClarinLicenseSubmissionUtils.java @@ -0,0 +1,122 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.submit.step; + +import java.sql.SQLException; +import java.util.List; +import java.util.Objects; + +import org.apache.commons.lang3.StringUtils; +import org.dspace.app.rest.exception.ClarinLicenseNotFoundException; +import org.dspace.authorize.AuthorizeException; +import org.dspace.content.Bitstream; +import org.dspace.content.Bundle; +import org.dspace.content.Item; +import org.dspace.content.clarin.ClarinLicense; +import org.dspace.content.factory.ClarinServiceFactory; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.ItemService; +import org.dspace.content.service.clarin.ClarinLicenseResourceMappingService; +import org.dspace.content.service.clarin.ClarinLicenseService; +import org.dspace.core.Constants; +import org.dspace.core.Context; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Shared logic for applying a CLARIN license selection to an in-progress + * submission item. Used by both the legacy top-level {@code /license} patch + * path in {@code WorkspaceItemRestRepository} and the section-scoped patch + * path handled by {@link ClarinLicenseResourceStep}. + * + * @author Milan Majchrak (milan.majchrak at dataquest.sk) + */ +public final class ClarinLicenseSubmissionUtils { + + private static final Logger log = LoggerFactory.getLogger(ClarinLicenseSubmissionUtils.class); + + private ClarinLicenseSubmissionUtils() { + } + + /** + * Apply the given CLARIN license selection to the supplied item. + *
    + *
  • Always clears the previously stored {@code dc.rights*} + * metadata and detaches any existing CLARIN license mapping + * from the bitstreams in the {@code ORIGINAL} bundle.
    • + *
  • If a non-blank {@code clarinLicenseName} is provided and + * resolves to an existing {@link ClarinLicense} the new + * license metadata is added to the item and the license is + * attached to every bitstream in the {@code ORIGINAL} bundle. + *
    • + *
+ * + * @param context DSpace context + * @param item the item being submitted + * @param clarinLicenseName name of the CLARIN license to apply, or + * {@code null}/empty to clear the current + * selection + * @throws SQLException on database errors + * @throws AuthorizeException on authorization errors + * @throws ClarinLicenseNotFoundException if a non-empty name was + * supplied but no matching CLARIN license exists + */ + public static void applyLicense(Context context, Item item, String clarinLicenseName) + throws SQLException, AuthorizeException { + if (Objects.isNull(item)) { + log.info("Cannot apply CLARIN license, item is null."); + return; + } + + ClarinLicenseService clarinLicenseService = + ClarinServiceFactory.getInstance().getClarinLicenseService(); + ClarinLicenseResourceMappingService clarinLicenseResourceMappingService = + ClarinServiceFactory.getInstance().getClarinLicenseResourceMappingService(); + ItemService itemService = ContentServiceFactory.getInstance().getItemService(); + + // Resolve license up-front so we fail before mutating state + ClarinLicense clarinLicense = null; + if (StringUtils.isNotBlank(clarinLicenseName)) { + clarinLicense = clarinLicenseService.findByName(context, clarinLicenseName); + if (Objects.isNull(clarinLicense)) { + throw new ClarinLicenseNotFoundException( + "The CLARIN license with name: " + clarinLicenseName + + " isn't supported in the CLARIN/DSpace"); + } + } + + // Clear existing license metadata from the item + clarinLicenseService.clearLicenseMetadataFromItem(context, item); + + // Detach existing CLARIN licenses from the uploaded bitstreams + List bundles = item.getBundles(Constants.CONTENT_BUNDLE_NAME); + for (Bundle bundle : bundles) { + for (Bitstream bitstream : bundle.getBitstreams()) { + clarinLicenseResourceMappingService.detachLicenses(context, bitstream); + } + } + + if (Objects.isNull(clarinLicense)) { + // Persist the cleared metadata state and stop. + itemService.update(context, item); + log.info("CLARIN license selection cleared on item {}.", item.getID()); + return; + } + + // Attach the new CLARIN license to every bitstream and add metadata + clarinLicenseService.addLicenseMetadataToItem(context, clarinLicense, item); + for (Bundle bundle : bundles) { + for (Bitstream bitstream : bundle.getBitstreams()) { + clarinLicenseResourceMappingService.attachLicense(context, clarinLicense, bitstream); + } + } + // Persist all metadata changes in a single update at the end. + itemService.update(context, item); + log.info("CLARIN license '{}' applied to item {}.", clarinLicenseName, item.getID()); + } +} diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinWorkspaceItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinWorkspaceItemRestRepositoryIT.java index fdc406c5b67f..aae481283540 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinWorkspaceItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinWorkspaceItemRestRepositoryIT.java @@ -820,6 +820,241 @@ public void updateClarinLicenseInWI() throws Exception { .andExpect(jsonPath("$.bitstreams", is(1))); } + /** + * Applying the CLARIN license via the section-scoped path + * `/sections/clarin-license/select` must work exactly like the legacy + * top-level `/license` path. + */ + @Test + public void addClarinLicenseViaSectionPatch() throws Exception { + context.turnOffAuthorisationSystem(); + WorkspaceItem witem = createWorkspaceItemWithFile(); + + String clarinLicenseName = "Test Section Clarin License"; + ClarinLicense clarinLicense = createClarinLicense(clarinLicenseName, "Test Def", "Test R Info", + Confirmation.NOT_REQUIRED); + context.restoreAuthSystemState(); + + List replaceOperations = new ArrayList(); + Map licenseReplaceOpValue = new HashMap(); + licenseReplaceOpValue.put("value", clarinLicenseName); + replaceOperations.add(new ReplaceOperation("/sections/clarin-license/select", + licenseReplaceOpValue)); + String updateBody = getPatchContent(replaceOperations); + + String tokenAdmin = getAuthToken(admin.getEmail(), password); + getClient(tokenAdmin).perform(patch("/api/submission/workspaceitems/" + witem.getID()) + .content(updateBody) + .contentType(MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isOk()); + + // Item metadata `dc.rights` must contain the CLARIN license name + assertClarinLicenseMetadata(witem, "dc", "rights", null, clarinLicenseName, false); + + // Bitstream must be attached to the CLARIN license + getClient(tokenAdmin).perform(get("/api/core/clarinlicenses/" + clarinLicense.getID())) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.bitstreams", is(1))); + } + + /** + * After applying a CLARIN license, GET on the workspace item must expose + * distinct payloads for the standard `license` section + * (CC license: url/acceptanceDate/granted) and the `clarin-license` + * section (name/definition/label/granted from `dc.rights*`). + */ + @Test + public void getWorkspaceItemReturnsDistinctLicenseSections() throws Exception { + context.turnOffAuthorisationSystem(); + WorkspaceItem witem = createWorkspaceItemWithFile(); + + String clarinLicenseName = "Distinct Sections Clarin License"; + createClarinLicense(clarinLicenseName, "Test Def", "Test R Info", + Confirmation.NOT_REQUIRED); + context.restoreAuthSystemState(); + + // Apply the CLARIN license through the section-scoped path + List replaceOperations = new ArrayList(); + Map licenseReplaceOpValue = new HashMap(); + licenseReplaceOpValue.put("value", clarinLicenseName); + replaceOperations.add(new ReplaceOperation("/sections/clarin-license/select", + licenseReplaceOpValue)); + String updateBody = getPatchContent(replaceOperations); + + String tokenAdmin = getAuthToken(admin.getEmail(), password); + getClient(tokenAdmin).perform(patch("/api/submission/workspaceitems/" + witem.getID()) + .content(updateBody) + .contentType(MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isOk()); + + // GET the workspace item and assert the two sections are distinct + getClient(tokenAdmin).perform(get("/api/submission/workspaceitems/" + witem.getID())) + .andExpect(status().isOk()) + // clarin-license section reflects CLARIN-specific fields + .andExpect(jsonPath("$.sections['clarin-license'].name", is(clarinLicenseName))) + .andExpect(jsonPath("$.sections['clarin-license'].definition").isNotEmpty()) + .andExpect(jsonPath("$.sections['clarin-license'].label").isNotEmpty()) + .andExpect(jsonPath("$.sections['clarin-license'].granted", is(true))) + .andExpect(jsonPath("$.sections.license.granted", is(false))) + .andExpect(jsonPath("$.sections.license.acceptanceDate").isEmpty()) + .andExpect(jsonPath("$.sections.license.url").isEmpty()); + } + + /** + * PATCH on `/sections/clarin-license/select` with an empty value must clear + * the previously selected license: `dc.rights*` metadata is removed and the + * license is detached from the uploaded bitstream. + */ + @Test + public void patchSelectWithEmptyValueClearsLicense() throws Exception { + context.turnOffAuthorisationSystem(); + WorkspaceItem witem = createWorkspaceItemWithFile(); + + String clarinLicenseName = "Empty Value Clarin License"; + ClarinLicense clarinLicense = createClarinLicense(clarinLicenseName, "Test Def", "Test R Info", + Confirmation.NOT_REQUIRED); + context.restoreAuthSystemState(); + + String tokenAdmin = getAuthToken(admin.getEmail(), password); + + // First select the license through the new section path + List ops = new ArrayList(); + Map selectValue = new HashMap(); + selectValue.put("value", clarinLicenseName); + ops.add(new ReplaceOperation("/sections/clarin-license/select", selectValue)); + getClient(tokenAdmin).perform(patch("/api/submission/workspaceitems/" + witem.getID()) + .content(getPatchContent(ops)) + .contentType(MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isOk()); + assertClarinLicenseMetadata(witem, "dc", "rights", null, clarinLicenseName, false); + getClient(tokenAdmin).perform(get("/api/core/clarinlicenses/" + clarinLicense.getID())) + .andExpect(jsonPath("$.bitstreams", is(1))); + + // Now clear the selection with an empty value + ops.clear(); + Map emptyValue = new HashMap(); + emptyValue.put("value", ""); + ops.add(new ReplaceOperation("/sections/clarin-license/select", emptyValue)); + getClient(tokenAdmin).perform(patch("/api/submission/workspaceitems/" + witem.getID()) + .content(getPatchContent(ops)) + .contentType(MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isOk()); + + // Item metadata cleared and license detached from bitstream + assertClarinLicenseMetadata(witem, "dc", "rights", null, null, true); + getClient(tokenAdmin).perform(get("/api/core/clarinlicenses/" + clarinLicense.getID())) + .andExpect(jsonPath("$.bitstreams", is(0))); + } + + /** + * Two consecutive PATCHes on `/sections/clarin-license/select` must replace + * the previously selected license: `dc.rights` reflects the latest name and + * the bitstream is moved from the first license to the second one. + */ + @Test + public void patchSelectReplacesPreviousLicense() throws Exception { + context.turnOffAuthorisationSystem(); + WorkspaceItem witem = createWorkspaceItemWithFile(); + + String firstName = "First Clarin License"; + String secondName = "Second Clarin License"; + ClarinLicense first = createClarinLicense(firstName, "Def1", "Info1", Confirmation.NOT_REQUIRED); + ClarinLicense second = createClarinLicense(secondName, "Def2", "Info2", Confirmation.NOT_REQUIRED); + context.restoreAuthSystemState(); + + String tokenAdmin = getAuthToken(admin.getEmail(), password); + + // Select first license + List ops = new ArrayList(); + Map v1 = new HashMap(); + v1.put("value", firstName); + ops.add(new ReplaceOperation("/sections/clarin-license/select", v1)); + getClient(tokenAdmin).perform(patch("/api/submission/workspaceitems/" + witem.getID()) + .content(getPatchContent(ops)) + .contentType(MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isOk()); + assertClarinLicenseMetadata(witem, "dc", "rights", null, firstName, false); + getClient(tokenAdmin).perform(get("/api/core/clarinlicenses/" + first.getID())) + .andExpect(jsonPath("$.bitstreams", is(1))); + + // Replace with second license through the same section path + ops.clear(); + Map v2 = new HashMap(); + v2.put("value", secondName); + ops.add(new ReplaceOperation("/sections/clarin-license/select", v2)); + getClient(tokenAdmin).perform(patch("/api/submission/workspaceitems/" + witem.getID()) + .content(getPatchContent(ops)) + .contentType(MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isOk()); + + assertClarinLicenseMetadata(witem, "dc", "rights", null, secondName, false); + getClient(tokenAdmin).perform(get("/api/core/clarinlicenses/" + first.getID())) + .andExpect(jsonPath("$.bitstreams", is(0))); + getClient(tokenAdmin).perform(get("/api/core/clarinlicenses/" + second.getID())) + .andExpect(jsonPath("$.bitstreams", is(1))); + } + + /** + * PATCH on `/sections/clarin-license/select` with a name that does not + * resolve to an existing CLARIN license must fail with 422 and must not + * mutate the item's `dc.rights*` metadata. + */ + @Test + public void patchSelectWithUnknownLicenseNameFails() throws Exception { + context.turnOffAuthorisationSystem(); + WorkspaceItem witem = createWorkspaceItemWithFile(); + List rightsBefore = itemService.getMetadata(witem.getItem(), "dc", "rights", null, + null, Item.ANY); + List rightsBeforeValues = new ArrayList(); + for (MetadataValue metadataValue : rightsBefore) { + rightsBeforeValues.add(metadataValue.getValue()); + } + context.restoreAuthSystemState(); + + List ops = new ArrayList(); + Map v = new HashMap(); + v.put("value", "Definitely Not An Existing License"); + ops.add(new ReplaceOperation("/sections/clarin-license/select", v)); + + String tokenAdmin = getAuthToken(admin.getEmail(), password); + getClient(tokenAdmin).perform(patch("/api/submission/workspaceitems/" + witem.getID()) + .content(getPatchContent(ops)) + .contentType(MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isUnprocessableEntity()); + + witem = context.reloadEntity(witem); + List rightsAfter = itemService.getMetadata(witem.getItem(), "dc", "rights", null, + null, Item.ANY); + List rightsAfterValues = new ArrayList(); + for (MetadataValue metadataValue : rightsAfter) { + rightsAfterValues.add(metadataValue.getValue()); + } + Assert.assertEquals(rightsBeforeValues, rightsAfterValues); + } + + /** + * Anonymous PATCH on `/sections/clarin-license/select` must be rejected + * (the user is not authenticated to modify the submission). + */ + @Test + public void patchSelectAsAnonymousIsUnauthorized() throws Exception { + context.turnOffAuthorisationSystem(); + WorkspaceItem witem = createWorkspaceItemWithFile(); + String clarinLicenseName = "Anon Clarin License"; + createClarinLicense(clarinLicenseName, "Def", "Info", Confirmation.NOT_REQUIRED); + context.restoreAuthSystemState(); + + List ops = new ArrayList(); + Map v = new HashMap(); + v.put("value", clarinLicenseName); + ops.add(new ReplaceOperation("/sections/clarin-license/select", v)); + + getClient().perform(patch("/api/submission/workspaceitems/" + witem.getID()) + .content(getPatchContent(ops)) + .contentType(MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isUnauthorized()); + } + /** * Create Item with standard handle. The handle definition for every community is configured * by the `lr.pid.community.configurations` properties. From 6523c283cbd722d7c3c540ef93f103f287e00ab6 Mon Sep 17 00:00:00 2001 From: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> Date: Wed, 27 May 2026 14:54:50 +0200 Subject: [PATCH 26/41] UFAL/Fix clarin-license IT: expect 422 for empty value on select patch (#1323) * Expect 422 for empty value on clarin-license select patch * Treat blank value as clear on clarin-license section select --- .../submit/step/ClarinLicenseResourceStep.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/step/ClarinLicenseResourceStep.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/step/ClarinLicenseResourceStep.java index b3e315af445e..9568f13942ca 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/step/ClarinLicenseResourceStep.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/submit/step/ClarinLicenseResourceStep.java @@ -12,7 +12,6 @@ import com.fasterxml.jackson.databind.JsonNode; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.dspace.app.rest.exception.ClarinLicenseNotFoundException; import org.dspace.app.rest.exception.UnprocessableEntityException; import org.dspace.app.rest.model.patch.JsonValueEvaluator; @@ -87,14 +86,11 @@ public void doPatchProcessing(Context context, HttpServletRequest currentRequest "The operation '" + op.getOp() + "' is not supported for path " + path); } String licenseName = extractLicenseName(op); - // Section endpoint: a missing or blank license name is treated as a - // client error (422). The legacy `/license` path in - // WorkspaceItemRestRepository intentionally treats a blank value as - // "clear the current license" for backwards compatibility. - if (StringUtils.isBlank(licenseName)) { - throw new UnprocessableEntityException( - "The patch value for path " + path + " must contain a non-empty license name."); - } + // A blank/missing license name is intentionally treated as a request + // to clear the currently selected CLARIN license, mirroring the + // legacy `/license` path in WorkspaceItemRestRepository. + // {@link ClarinLicenseSubmissionUtils#applyLicense} handles a blank + // name as a "clear" operation. try { ClarinLicenseSubmissionUtils.applyLicense(context, source.getItem(), licenseName); } catch (ClarinLicenseNotFoundException ex) { From 82085cba05007273408482a116e551a069593cc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Wed, 3 Jun 2026 11:49:40 +0200 Subject: [PATCH 27/41] UFAL/Fix refbox buttons (ufal/clarin-dspace#1367) (#1318) * adding test and fixing the issue fixes ufal/clarin-dspace#1366 there was a conflict between the produces=application/json and response.setContentType("application/xml") * Strengthen citations endpoint test assertions for ufal/clarin-dspace#1366 regression coverage Agent-Logs-Url: https://github.com/ufal/clarin-dspace/sessions/e385ef9e-8186-4899-b811-fc82bbfa942b --------- (cherry picked from commit 8400982afc50331c6fa6a94fa39035ee09de92af) Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: kosarko <1842385+kosarko@users.noreply.github.com> --- .../app/rest/ClarinRefBoxController.java | 1 - .../app/rest/ClarinRefBoxControllerIT.java | 130 +++++++++++++++++- 2 files changed, 129 insertions(+), 2 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/ClarinRefBoxController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/ClarinRefBoxController.java index 3e889361609a..8767b95b64a7 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/ClarinRefBoxController.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/ClarinRefBoxController.java @@ -250,7 +250,6 @@ public ResponseEntity getCitationText(@RequestParam(name = "type") String type, // Some preparing for the getting the data. OAIRequestParameters parameters = new OAIRequestParameters(parameterMap); - response.setContentType("application/xml"); // Get the OAI-PMH data. oaipmh = dataProvider.handle(parameters); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinRefBoxControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinRefBoxControllerIT.java index ce87f39bff7d..aa767b9895b0 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinRefBoxControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinRefBoxControllerIT.java @@ -7,12 +7,19 @@ */ package org.dspace.app.rest; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.emptyOrNullString; import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.not; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.app.rest.utils.SolrOAIReindexer; import org.dspace.app.rest.utils.Utils; import org.dspace.builder.CollectionBuilder; import org.dspace.builder.CommunityBuilder; @@ -20,25 +27,79 @@ import org.dspace.content.Collection; import org.dspace.content.Item; import org.dspace.services.ConfigurationService; +import org.dspace.solr.MockSolrServer; +import org.dspace.xoai.services.api.cache.XOAICacheService; +import org.dspace.xoai.services.api.solr.SolrServerResolver; +import org.dspace.xoai.services.api.xoai.ItemRepositoryResolver; +import org.junit.After; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.util.ReflectionTestUtils; /** * The Integration Test class for the ClarinRefBoxController. */ +@TestPropertySource(properties = {"oai.enabled = true"}) public class ClarinRefBoxControllerIT extends AbstractControllerIntegrationTest { @Autowired ConfigurationService configurationService; + @Autowired + SolrOAIReindexer solrOAIReindexer; + + // Mock OAI cache to disable it during tests (avoids side-effects from cache state) + @MockBean + private XOAICacheService xoaiCacheService; + + // Mock the OAI SolrServerResolver — overridden to return an embedded Solr client in setUp() + @MockBean + private SolrServerResolver solrServerResolver; + + // Used to reset the cached DSpaceItemSolrRepository before each test + @Autowired(required = false) + private ItemRepositoryResolver itemRepositoryResolver; + + private MockSolrServer mockOAISolr; + // FS = featuredService private Item itemWithFS; private Item item; private Collection collection; + @Override @Before - public void setup() { + public void setUp() throws Exception { + super.setUp(); + + // Skip all tests if the OAI module is not on the classpath + try { + Class.forName("org.dspace.app.configuration.OAIWebConfig"); + } catch (ClassNotFoundException ce) { + Assume.assumeNoException(ce); + } + + // Initialise embedded Solr for the OAI core + mockOAISolr = new MockSolrServer("oai"); + when(solrServerResolver.getServer()).thenReturn(mockOAISolr.getSolrServer()); + + // Disable OAI caching so tests see live Solr state + when(xoaiCacheService.isActive()).thenReturn(false); + when(xoaiCacheService.hasCache(anyString())).thenReturn(false); + + // Reset the cached ItemRepository so it is re-created with the embedded client + if (itemRepositoryResolver != null) { + ReflectionTestUtils.setField(itemRepositoryResolver, "itemRepository", null); + } + + // Ensure the reindexer also uses the embedded Solr client + ReflectionTestUtils.setField(solrOAIReindexer, "solrServerResolver", solrServerResolver); + context.turnOffAuthorisationSystem(); parentCommunity = CommunityBuilder.createCommunity(context).withName("test").build(); collection = CollectionBuilder.createCollection(context, parentCommunity).withName("Collection 1").build(); @@ -58,6 +119,17 @@ public void setup() { .withMetadata("local","featuredService","pmltq", "Arabic|URLArabic") .build(); context.restoreAuthSystemState(); + + // Index the test item in the xoai Solr core so the citations endpoint can find it. + solrOAIReindexer.reindexItem(item); + } + + @After + public void tearDownOAI() throws Exception { + if (mockOAISolr != null) { + mockOAISolr.destroy(); + mockOAISolr = null; + } } @Test @@ -325,4 +397,60 @@ public void testDisplayTextWithMoreThanFiveAuthors() throws Exception { .andExpect(jsonPath("$.displayText").value(org.hamcrest.Matchers.containsString( "First Author; et al."))); } + + // --- Citations endpoint tests (#1366) --- + + @Test + public void testCitationsEndpointReturnsBibtex() throws Exception { + // Verifies the /citations endpoint does not return a 500 for a valid handle + bibtex + // type (regression test for #1366) and that the response is a valid OaiMetadataWrapper. + // Content-Type must be application/json (catches reintroduction of the XML preset bug). + // $.metadata must contain "@misc{" which is the BibTeX entry marker produced by the XSLT. + getClient().perform(get("/api/core/refbox/citations") + .param("type", "bibtex") + .param("handle", item.getHandle())) + .andExpect(status().isOk()) + .andExpect(content().contentTypeCompatibleWith(MediaType.APPLICATION_JSON)) + .andExpect(jsonPath("$.metadata", containsString("@misc{"))); + } + + @Test + public void testCitationsEndpointReturnsCmdi() throws Exception { + // Verifies the /citations endpoint does not return a 500 for a valid handle + cmdi type + // (regression test for #1366) and that the response is a valid OaiMetadataWrapper. + // Content-Type must be application/json (catches reintroduction of the XML preset bug). + // $.metadata must be non-empty (the CMDI crosswalk always produces XML content). + getClient().perform(get("/api/core/refbox/citations") + .param("type", "cmdi") + .param("handle", item.getHandle())) + .andExpect(status().isOk()) + .andExpect(content().contentTypeCompatibleWith(MediaType.APPLICATION_JSON)) + .andExpect(jsonPath("$.metadata", not(emptyOrNullString()))); + } + + @Test + public void testCitationsEndpointWithUrlBuiltHandle() throws Exception { + // Reproduces the exact URL format that buildExportFormats() produces, where + // the handle is the canonical URL path (e.g. "/hdl.handle.net/123456789/xxx"). + // The endpoint must not return 500 for this input. + String handle = "/" + Utils.getCanonicalHandleUrlNoProtocol(item); + getClient().perform(get("/api/core/refbox/citations") + .param("type", "bibtex") + .param("handle", handle)) + .andExpect(status().isOk()); + } + + @Test + public void testCitationsEndpointWithMissingTypeParam() throws Exception { + getClient().perform(get("/api/core/refbox/citations") + .param("handle", item.getHandle())) + .andExpect(status().is4xxClientError()); + } + + @Test + public void testCitationsEndpointWithMissingHandleParam() throws Exception { + getClient().perform(get("/api/core/refbox/citations") + .param("type", "bibtex")) + .andExpect(status().is4xxClientError()); + } } From 3dc9cecf7de92c8c1bcec99461add5aaaa827b6a Mon Sep 17 00:00:00 2001 From: Kasinhou <129340513+Kasinhou@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:58:45 +0200 Subject: [PATCH 28/41] Health report, report diff fixes (#1254) * fix(health-report): fix CLI args, multi-check support, and report-diff comparison logic * Fix ReportDiff setup and date validation * fix failed integration test * added tests, used -c 1 2 instead of -c 1 -c 2 * used UNLIMITED_VALUES unstead of MAX_VALUE * improved doc * used multilist for -c , removed unused method * improved doc * WIP updated health report and report diff * Complete update of health-report and report-diff * Sorting reports and updating tests, plus enable multiple -c in report-diff * Improved docs, output and info and logic * Improved comparision of reports w/o changes, or with only one report specified * Updated tests * Removed unused Report and refactor getChecks * Address review comments: simplify null checks, deduplicate skipped-checks section, validate report IDs before defaulting Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Matus Kasak Co-authored-by: Paurikova2 Co-authored-by: milanmajchrak Co-authored-by: Claude Fable 5 --- .../dspace/app/healthreport/HealthReport.java | 149 ++- .../HealthReportScriptConfiguration.java | 21 +- .../org/dspace/app/reportdiff/ReportDiff.java | 890 ++++++++++++++---- .../ReportDiffScriptConfiguration.java | 31 +- .../content/ReportResultServiceImpl.java | 6 - .../dspace/content/dao/ReportResultDAO.java | 11 - .../content/dao/impl/ReportResultDAOImpl.java | 14 - .../content/service/ReportResultService.java | 10 - .../main/java/org/dspace/health/Report.java | 229 ----- .../main/resources/report-diff-fields.json | 96 +- .../org/dspace/scripts/HealthReportIT.java | 137 ++- .../java/org/dspace/scripts/ReportDiffIT.java | 442 +++++++-- .../rest/repository/ScriptRestRepository.java | 10 +- dspace/config/launcher.xml | 8 +- 14 files changed, 1428 insertions(+), 626 deletions(-) delete mode 100644 dspace-api/src/main/java/org/dspace/health/Report.java diff --git a/dspace-api/src/main/java/org/dspace/app/healthreport/HealthReport.java b/dspace-api/src/main/java/org/dspace/app/healthreport/HealthReport.java index 36bb15c1056f..d450a2235b45 100644 --- a/dspace-api/src/main/java/org/dspace/app/healthreport/HealthReport.java +++ b/dspace-api/src/main/java/org/dspace/app/healthreport/HealthReport.java @@ -13,10 +13,14 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Date; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import javax.mail.MessagingException; import org.apache.commons.cli.Option; @@ -29,10 +33,11 @@ import org.dspace.core.Context; import org.dspace.core.Email; import org.dspace.core.I18nUtil; +import org.dspace.core.factory.CoreServiceFactory; +import org.dspace.core.service.PluginService; import org.dspace.eperson.factory.EPersonServiceFactory; import org.dspace.eperson.service.EPersonService; import org.dspace.health.Check; -import org.dspace.health.Report; import org.dspace.health.ReportInfo; import org.dspace.scripts.DSpaceRunnable; import org.dspace.services.ConfigurationService; @@ -56,12 +61,12 @@ public class HealthReport extends DSpaceRunnable checks = Report.checks(); + private static final LinkedHashMap checks = getChecks(); /** - * `-i`: Info, show help information. + * `-h`: Help, show help information. */ - private boolean info = false; + private boolean help = false; /** * `-e`: Email, send report to specified email address. @@ -69,9 +74,10 @@ public class HealthReport extends DSpaceRunnable specificChecks = new ArrayList<>(); /** * `-f`: For, specify the last N days to consider. @@ -80,9 +86,9 @@ public class HealthReport extends DSpaceRunnable= getNumberOfChecks()) { - specificCheck = -1; + String[] checkOptions = commandLine.getOptionValues('c'); + for (String checkOption : checkOptions) { + try { + int checkIndex = Integer.parseInt(checkOption); + if (checkIndex < 0 || checkIndex >= getNumberOfChecks()) { + handler.logError("Invalid value for check: " + checkOption + + ". Must be an integer from 0 to " + (getNumberOfChecks() - 1) + "."); + throw new ParseException("Invalid check index: " + checkOption); + } + specificChecks.add(checkIndex); + } catch (NumberFormatException e) { + handler.logError("Invalid value for check: '" + checkOption + + "'. It has to be an integer number from 0 to " + (getNumberOfChecks() - 1) + "."); + throw new ParseException("Invalid check value: " + checkOption); } - } catch (NumberFormatException e) { - log.info("Invalid value for check. It has to be a number from the displayed range."); - return; } } - // `-f`: For, specify the last N days to consider. + // `-f`: For, specify the last N days to consider. Must be a positive integer. if (commandLine.hasOption('f')) { String daysOption = commandLine.getOptionValue('f'); try { forLastNDays = Integer.parseInt(daysOption); + if (forLastNDays <= 0) { + handler.logError("Invalid value for -f: " + daysOption + + ". Must be a positive integer (greater than 0)."); + throw new ParseException("Invalid -f value: " + daysOption); + } } catch (NumberFormatException e) { - log.info("Invalid value for last N days. Argument f has to be a number."); - return; + handler.logError("Invalid value for -f: '" + daysOption + + "'. Must be a positive integer."); + throw new ParseException("Invalid -f value: " + daysOption); } } - // `-o`: Output, specify a file to save the report. - if (commandLine.hasOption('o')) { - fileName = commandLine.getOptionValue('o'); + // `-r`: Report, specify a file to save the report. + if (commandLine.hasOption('r')) { + reportFile = commandLine.getOptionValue('r'); } } @Override public void internalRun() throws Exception { - if (info) { + if (help) { printHelp(); return; } @@ -149,15 +167,14 @@ public void internalRun() throws Exception { ReportInfo ri = new ReportInfo(this.forLastNDays); StringBuilder sbReport = new StringBuilder(); - sbReport.append("\n\nHEALTH REPORT:\n"); int position = -1; JSONObject root = new JSONObject(); // Create the array JSONArray checksArray = new JSONArray(); - for (Map.Entry check_entry : Report.checks().entrySet()) { + for (Map.Entry check_entry : checks.entrySet()) { ++position; - if (specificCheck != -1 && specificCheck != position) { + if (!specificChecks.isEmpty() && !specificChecks.contains(position)) { continue; } @@ -197,10 +214,14 @@ public void internalRun() throws Exception { reportResultService.update(context, reportResult); context.commit(); + // Prepend the header with the persisted report ID so users can refer to it later + String finalReport = "\n\nHEALTH REPORT " + reportResult.getID() + ":\n" + sbReport.toString(); + // save output to file - if (fileName != null) { - InputStream inputStream = toInputStream(sbReport.toString(), StandardCharsets.UTF_8); - handler.writeFilestream(context, fileName, inputStream, "export"); + if (reportFile != null) { + InputStream inputStream = toInputStream(finalReport, StandardCharsets.UTF_8); + handler.writeFilestream(context, reportFile, inputStream, "export"); + context.commit(); context.restoreAuthSystemState(); @@ -213,27 +234,34 @@ public void internalRun() throws Exception { for (String recipient : emails) { e.addRecipient(recipient); } - e.addArgument(sbReport.toString()); + e.addArgument(finalReport); e.send(); + handler.logInfo("Report sent to: " + String.join(", ", emails)); } catch (IOException | MessagingException e) { log.error("Error sending email:", e); + handler.logError("Error sending email to " + String.join(", ", emails) + + ": " + e.getMessage()); } } - handler.logInfo(sbReport.toString()); + handler.logInfo(finalReport); } } @Override public void printHelp() { - handler.logInfo("\n\nINFORMATION\nThis process creates a health report of your DSpace.\n" + + int configuredForLastNDays = configurationService.getIntProperty("healthcheck.last_n_days"); + handler.logInfo("\n\nHELP\nThis process creates a health report of your DSpace.\n" + "You can choose from these available options:\n" + - " -i, --info Show help information\n" + + " -h, --help Show help information\n" + " -e, --email Send report to specified email address\n" + - " -c, --check Perform only specific check by index (0-" + (getNumberOfChecks() - 1) + ")\n" + - " -f, --for Specify the last N days to consider\n" + - " -o, --output Specify a file to save the report\n\n" + - "If you want to execute only one check using -c, use check index:\n" + checksNamesToString() + "\n" + " -c, --check Perform specific check(s) by index (0-" + (getNumberOfChecks() - 1) + + "). Repeat the flag (e.g. -c 1 -c 3) to run multiple checks. " + + "Default: All checks\n" + + " -f, --for Specify the last N days to consider (positive integer). " + + "Default: " + configuredForLastNDays + "\n" + + " -r, --report Specify a file to save the report\n\n" + + "Available checks:\n" + checksNamesToString() + "\n" ); } @@ -242,13 +270,21 @@ public void printHelp() { * This method is used to print the options used for the report. */ private String printCommandlineOptions() { - // Return key-value pairs of options StringBuilder options = new StringBuilder(); + Set processedOptions = new LinkedHashSet<>(); + for (Option option : commandLine.getOptions()) { String key = option.getOpt(); - String value = commandLine.getOptionValue(key); - if (value != null) { - options.append(String.format(" -%s: %s\n", key, value)); + if (key == null || processedOptions.contains(key)) { + continue; + } + processedOptions.add(key); + + String[] values = commandLine.getOptionValues(key); + if (values != null && values.length > 0) { + for (String value : values) { + options.append(String.format(" -%s: %s\n", key, value)); + } } else { options.append(String.format(" -%s\n", key)); } @@ -295,4 +331,25 @@ public static String getCheckName(int specificCheck) { } return null; // should not happen } + + /** + * Create check list from configured healthcheck plugins. + */ + private static LinkedHashMap getChecks() { + LinkedHashMap loadedChecks = new LinkedHashMap<>(); + String[] checkNames = DSpaceServicesFactory.getInstance().getConfigurationService() + .getArrayProperty("healthcheck.checks"); + PluginService pluginService = CoreServiceFactory.getInstance().getPluginService(); + + for (String checkName : checkNames) { + Check check = (Check) pluginService.getNamedPlugin(Check.class, checkName); + if (check != null) { + loadedChecks.put(checkName, check); + } else { + log.warn("Could not find implementation for [{}]", checkName); + } + } + + return loadedChecks; + } } diff --git a/dspace-api/src/main/java/org/dspace/app/healthreport/HealthReportScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/app/healthreport/HealthReportScriptConfiguration.java index 771cc70aadb9..613b18977a5d 100644 --- a/dspace-api/src/main/java/org/dspace/app/healthreport/HealthReportScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/app/healthreport/HealthReportScriptConfiguration.java @@ -7,6 +7,7 @@ */ package org.dspace.app.healthreport; +import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.dspace.scripts.configuration.ScriptConfiguration; @@ -32,20 +33,24 @@ public void setDspaceRunnableClass(Class dspaceRunnableClass) { public Options getOptions() { if (options == null) { Options options = new Options(); - options.addOption("i", "info", false, + options.addOption("h", "help", false, "Show help information."); options.addOption("e", "email", true, "Send report to this email address."); options.getOption("e").setType(String.class); - options.addOption("c", "check", true, - String.format("Perform only specific check (use index from 0 to %d, " + - "otherwise perform default checks).", HealthReport.getNumberOfChecks() - 1)); - options.getOption("c").setType(String.class); + Option checkOption = Option.builder("c").longOpt("check").hasArgs() + .desc(String.format("Perform specific check(s) by index (0 to %d). " + + "Repeat the flag (e.g. -c 1 -c 3) to run multiple checks. " + + "Default: All checks.", + HealthReport.getNumberOfChecks() - 1)) + .type(String.class) + .build(); + options.addOption(checkOption); options.addOption("f", "for", true, - "Report for last N days. Used only in general information for now."); + "Report for last N days (positive integer). Used only in general information for now."); options.getOption("f").setType(String.class); - options.addOption("o", "output", true, - "Save report to the file."); + options.addOption("r", "report", true, + "Specify the report file to store the output."); super.options = options; } diff --git a/dspace-api/src/main/java/org/dspace/app/reportdiff/ReportDiff.java b/dspace-api/src/main/java/org/dspace/app/reportdiff/ReportDiff.java index eaaf7bee6035..c4e8a0255fdd 100644 --- a/dspace-api/src/main/java/org/dspace/app/reportdiff/ReportDiff.java +++ b/dspace-api/src/main/java/org/dspace/app/reportdiff/ReportDiff.java @@ -10,7 +10,6 @@ import java.io.IOException; import java.io.InputStream; import java.sql.SQLException; -import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -22,10 +21,14 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.mail.MessagingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.flipkart.zjsonpatch.JsonDiff; import org.apache.commons.cli.ParseException; import org.apache.logging.log4j.LogManager; @@ -57,39 +60,40 @@ public class ReportDiff extends DSpaceRunnable { private static final ObjectMapper mapper = new ObjectMapper(); - private ReportResultService reportResultService; - private EPersonService ePersonService; + private ReportResultService reportResultService = ContentServiceFactory.getInstance().getReportResultService(); + private EPersonService ePersonService = EPersonServiceFactory.getInstance().getEPersonService(); /** - * `-i`: Info, show help information. + * `-h`: Help, show help information. */ - private boolean info = false; + private boolean help = false; /** - * `-d`: Dates, show all dates that the report was generated for a specific check type. + * `-l`: List all stored reports with IDs, timestamps and arguments. */ - private boolean showDates = false; + private boolean showList = false; /** - * `-l`: Limits the number of report entries (dates) displayed when using the --date option. + * `-m`: Maximum number of report entries displayed when using --list. * Default is -1 (no limit). */ - private long limit = -1; + private long maxEntries = -1; /** - * `-c`: Check, perform only specific check by index (0-`getNumberOfChecks()`). + * `-c`: Check, perform only specific checks by index (0-`getNumberOfChecks()`). + * Supports multiple values. */ - private int specificCheck = -1; + private List specificChecks = new ArrayList<>(); /** - * `-f`: From, specify the start date for the report. + * `-s`: Source, specify source report ID. */ - private Date from = null; + private Integer sourceReportId = null; /** - * `-t`: Till, specify the end date for the report. + * `-t`: Till, specify target report ID. */ - private Date to = null; + private Integer targetReportId = null; /** * `-e`: Email, send report to specified email address. @@ -101,6 +105,8 @@ public class ReportDiff extends DSpaceRunnable { private static final String REPORT_DIFF_FIELDS = "report-diff-fields.json"; private static final String FIELD_MAPPINGS_KEY = "fieldMappings"; private static final String FIELD_ORDER_KEY = "fieldOrder"; + private static final Pattern SHORT_ARG_WITH_VALUE = Pattern.compile("^-([a-zA-Z]):\\s*(.*)$"); + private static final Pattern SHORT_ARG_WITHOUT_VALUE = Pattern.compile("^-([a-zA-Z])$"); // Field configuration cache private static Map fieldMappings = null; @@ -135,6 +141,11 @@ private void loadFieldConfiguration() { fieldOrder.add(fieldNode.asText()); } } + } else { + log.warn("Report diff fields configuration '{}' not found on the classpath. " + + "Field mappings will be empty.", REPORT_DIFF_FIELDS); + fieldMappings = new LinkedHashMap<>(); + fieldOrder = new ArrayList<>(); } } catch (IOException e) { log.error("Error loading report diff fields configuration '{}': {}. Using empty configuration.", @@ -157,67 +168,108 @@ public ReportDiffScriptConfiguration getScriptConfiguration() { @Override public void setup() throws ParseException { ePersonService = EPersonServiceFactory.getInstance().getEPersonService(); - reportResultService = ContentServiceFactory.getInstance().getReportResultService(); - // `-i`: Info, show help information. - if (commandLine.hasOption('i')) { - info = true; + // `-h`: Help, show help information. + if (commandLine.hasOption('h')) { + help = true; return; } - // `-c`: Check, perform only specific check by index (0-`getNumberOfChecks()`). + // `-c`: Check, perform only specific checks by index (0-`getNumberOfChecks()`). + // Supports multiple values e.g. -c 0 3 4 if (commandLine.hasOption('c')) { - specificCheck = parseCheckOption(commandLine.getOptionValue('c')); - if (specificCheck == -1) { - // Error already logged in parseCheckOption - return; + String[] checkOptions = commandLine.getOptionValues('c'); + for (String checkOption : checkOptions) { + int parsedCheck = parseCheckOption(checkOption); + if (parsedCheck == -1) { + handler.logWarning("Invalid value for -c: '" + checkOption + + "'. All checks will be compared."); + specificChecks.clear(); + break; + } + specificChecks.add(parsedCheck); } } - // `-d`: Dates, show all dates that the report was generated for a specific check type. - if (commandLine.hasOption('d')) { - showDates = true; - try { - if (commandLine.hasOption("l")) { - limit = Long.parseLong(commandLine.getOptionValue("l")); + // `-l`: List all available reports with IDs/timestamps/args. + if (commandLine.hasOption('l')) { + showList = true; + if (commandLine.hasOption('m')) { + String mValue = commandLine.getOptionValue('m'); + try { + long parsedMax = Long.parseLong(mValue); + if (parsedMax <= 0) { + handler.logWarning("Invalid value for -m: '" + mValue + + "'. Must be a positive integer. All entries will be shown."); + maxEntries = -1; + } else { + maxEntries = parsedMax; + } + } catch (NumberFormatException e) { + handler.logWarning("Invalid value for -m: '" + mValue + + "'. Must be a positive integer. All entries will be shown."); + maxEntries = -1; } - } catch (NumberFormatException e) { - handler.logError("Invalid value for -l. Must be a valid number."); - return; } } - // `-f`: From, specify the start date for the report. - from = parseDateOption(commandLine.getOptionValue('f')); - // `-t`: To, specify the end date for the report. - to = parseDateOption(commandLine.getOptionValue('t')); + // `-s`: Source, specify source report ID. + if (commandLine.hasOption('s')) { + String sValue = commandLine.getOptionValue('s'); + sourceReportId = parseReportIdOption(sValue); + if (sourceReportId == null) { + handler.logWarning("Invalid value for -s: '" + sValue + + "'. The last report from the database will be used instead."); + } + } + + // `-t`: Target, specify target report ID. + if (commandLine.hasOption('t')) { + String tValue = commandLine.getOptionValue('t'); + targetReportId = parseReportIdOption(tValue); + if (targetReportId == null) { + handler.logWarning("Invalid value for -t: '" + tValue + + "'. The last report from the database will be used instead."); + } + } if (commandLine.hasOption('e')) { emails = commandLine.getOptionValues('e'); - handler.logInfo("\nReport sent to this email address: " + String.join(", ", emails)); + handler.logInfo("\nReport will be sent to: " + String.join(", ", emails)); } } @Override public void internalRun() throws Exception { // If the user requested help information, we will display it. - if (info) { + if (help) { printHelp(); return; } // If the user requested to see all report dates, we will display them. - if (showDates) { + if (showList) { displayReportDates(); return; } try (Context context = new Context()) { - defaultDate(context); - - // If the user specified a specific check, we need to ensure that both `from` and `to` dates are set. - if (!validateDateRange()) { + // Validate the explicitly provided report IDs before defaulting the missing ones, + // so no defaulting message is logged when a provided report ID is invalid. + if (!validateReportIdSelection()) { return; } + if (!reportExists(context, sourceReportId) || !reportExists(context, targetReportId)) { + return; + } + + // If at least one of -s/-t is missing, fill missing values from latest reports. + if (sourceReportId == null || targetReportId == null) { + defaultReportIds(context); + if (sourceReportId == null || targetReportId == null) { + handler.logInfo("Need at least 2 reports in the database to perform a comparison. Aborting."); + return; + } + } // If the user specified a specific check, we will parse the dates and compare the reports. compareReports(context); @@ -236,66 +288,80 @@ private int parseCheckOption(String checkOption) { try { int index = Integer.parseInt(checkOption); if (index < 0 || index >= HealthReport.getNumberOfChecks()) { - handler.logError("Invalid value for check. Must be between 0 and " + - (HealthReport.getNumberOfChecks() - 1) + ". Using all checks."); return -1; } return index; } catch (NumberFormatException e) { - handler.logError("Invalid value for check. It must be a NUMBER from the displayed range."); return -1; } } /** - * Parse the date option and return a Date object. + * Parse report ID option and return an Integer. * If the option is invalid, log an error and return null. - * The date format is expected to be "yyyy-MM-dd HH:mm:ss.SSS". * * @param optionValue the date option value - * @return the parsed Date or null if invalid + * @return the parsed report ID or null if invalid */ - private Date parseDateOption(String optionValue) { + private Integer parseReportIdOption(String optionValue) { if (optionValue == null) { return null; } try { - LocalDateTime ldt = LocalDateTime.parse(optionValue, FORMATTER); - return Date.from(ldt.atZone(ZoneId.systemDefault()).toInstant()); + return Integer.parseInt(optionValue); } catch (Exception e) { - handler.logError("Cannot create a Date from the input: " + optionValue); return null; } } /** - * Validate the date range specified by `from` and `to`. - * If the dates are invalid, log an error and return false. - * If both dates are set, ensure that `to` is not before `from`. + * Validate the explicitly provided report IDs. + * The report IDs are optional (missing ones are defaulted later), + * but when provided they must be positive integers. * - * @return true if the date range is valid, false otherwise + * @return true if the provided report IDs are valid, false otherwise */ - private boolean validateDateRange() { - if (to != null && from != null && to.before(from)) { - handler.logError("The 'to' date cannot be before the 'from' date."); + private boolean validateReportIdSelection() { + if (sourceReportId != null && sourceReportId <= 0) { + handler.logError("The 'source' report ID must be a positive integer."); return false; - } else if (Objects.isNull(from) || Objects.isNull(to)) { - handler.logError("Both 'from' and 'to' dates must be specified when using a specific check."); + } + + if (targetReportId != null && targetReportId <= 0) { + handler.logError("The 'target' report ID must be a positive integer."); return false; } + return true; } /** - * Sets default values for the `from` and `to` dates if they are not already specified. + * Check that an explicitly provided report ID exists in the database. + * A {@code null} report ID is considered valid because it is defaulted later. * - * @param context the application context used for fetching reports and logging + * @param context the application context + * @param reportId the report ID to check, may be null + * @return true if the report ID is null or the report exists, false otherwise + * @throws SQLException if a database error occurs */ - private void defaultDate(Context context) { - if (Objects.nonNull(from) && Objects.nonNull(to)) { - return; + private boolean reportExists(Context context, Integer reportId) throws SQLException { + if (reportId == null) { + return true; + } + if (reportResultService.find(context, reportId) == null) { + handler.logInfo("No report found for report ID: " + reportId); + return false; } - handler.logInfo("No dates specified, using the last two dates from the database."); + return true; + } + + /** + * Sets default values for the source and target report IDs if not already specified. + * + * @param context the application context used for fetching reports and logging + */ + private void defaultReportIds(Context context) { + boolean bothMissing = sourceReportId == null && targetReportId == null; try { List allReports = reportResultService.findAll(context); @@ -304,13 +370,36 @@ private void defaultDate(Context context) { return; } + // findAll() does not guarantee ordering; sort by lastModified ascending so the + // newest reports are at the end of the list. + allReports.sort(Comparator.comparing(ReportResult::getLastModified)); int size = allReports.size(); - if (Objects.isNull(to) && size > 0) { - to = allReports.get(size - 1).getLastModified(); + if (bothMissing) { + handler.logInfo("No report IDs specified, using the last two reports from the database."); + if (size > 0) { + targetReportId = allReports.get(size - 1).getID(); + } + if (size > 1) { + sourceReportId = allReports.get(size - 2).getID(); + } + return; + } + + if (sourceReportId == null) { + handler.logInfo("Only '-t' was specified; '-s' will be set to the latest report from the " + + "database."); + if (size > 0) { + sourceReportId = allReports.get(size - 1).getID(); + } } - if (Objects.isNull(from) && size > 1) { - from = allReports.get(size - 2).getLastModified(); + + if (targetReportId == null) { + handler.logInfo("Only '-s' was specified; '-t' will be set to the latest report from the " + + "database."); + if (size > 0) { + targetReportId = allReports.get(size - 1).getID(); + } } } catch (SQLException e) { throw new RuntimeException(e); @@ -318,17 +407,24 @@ private void defaultDate(Context context) { } /** - * Display all report dates for the specified check type. + * Display available reports with their IDs and timestamps. * If no reports are found, log an appropriate message. - * Display the last 20 report dates for each type, sorted by date. - * In the format "Report Type: \n - | \n", + * Display the last 20 report entries for each type, sorted by date. + * In the format "Report Type: \n - ID: | | \n", */ private void displayReportDates() { try (Context context = new Context()) { context.setCurrentUser(ePersonService.find(context, getEpersonIdentifier())); List allReports = reportResultService.findAll(context); - // Determine how many reports to process, respecting the `limit` if it's within valid range - long limitCount = (limit > 0 && limit < allReports.size()) ? limit : allReports.size(); + if (allReports == null || allReports.isEmpty()) { + handler.logInfo("No reports found in the database."); + return; + } + // findAll() does not guarantee ordering; sort by lastModified ascending so the + // newest reports are at the end of the list. + allReports.sort(Comparator.comparing(ReportResult::getLastModified)); + // Determine how many reports to process, respecting maxEntries if it's within valid range + long limitCount = (maxEntries > 0 && maxEntries < allReports.size()) ? maxEntries : allReports.size(); Map> reportDatesMap = new HashMap<>(); for (long i = 0; i < limitCount; i++) { // the newest report is at the end of the list, so we reverse the index @@ -337,10 +433,10 @@ private void displayReportDates() { .toInstant() .atZone(ZoneId.systemDefault()).toLocalDateTime()); reportDatesMap.computeIfAbsent(report.getType(), k -> new ArrayList<>()) - .add(new DateWithArgs(formattedDate, report.getArgs())); + .add(new DateWithArgs(report.getID(), formattedDate, report.getArgs())); } - StringBuilder sb = new StringBuilder("Report Dates Summary:\n"); + StringBuilder sb = new StringBuilder("Available Reports Summary:\n"); reportDatesMap.forEach((type, dates) -> { sb.append("Report Type: ").append(type).append("\n"); dates.stream() @@ -348,9 +444,11 @@ private void displayReportDates() { .limit(20) .forEach(dwa -> sb .append(" - ") + .append("ID: ").append(dwa.getId()) + .append(" | ") .append(dwa.getDate()) .append(" | ") - .append(dwa.getArgs() != null ? dwa.getArgs().strip() : "") + .append(formatReportArgsForDisplay(dwa.getArgs())) .append("\n")); }); @@ -360,10 +458,90 @@ private void displayReportDates() { } } + private String formatReportArgsForDisplay(String args) { + if (args == null || args.isBlank()) { + return ""; + } + + List formattedEntries = new ArrayList<>(); + String[] lines = args.split("\\r?\\n"); + for (String line : lines) { + String trimmed = line.trim(); + if (trimmed.isEmpty()) { + continue; + } + + Matcher withValue = SHORT_ARG_WITH_VALUE.matcher(trimmed); + if (withValue.matches()) { + String shortOpt = withValue.group(1); + String value = withValue.group(2); + String longOpt = resolveHealthReportLongOption(shortOpt); + + if ("c".equals(shortOpt)) { + value = appendCheckName(value); + } + + if (longOpt != null) { + formattedEntries.add("--" + longOpt + ": " + value); + } else { + formattedEntries.add(trimmed); + } + continue; + } + + Matcher withoutValue = SHORT_ARG_WITHOUT_VALUE.matcher(trimmed); + if (withoutValue.matches()) { + String shortOpt = withoutValue.group(1); + String longOpt = resolveHealthReportLongOption(shortOpt); + if (longOpt != null) { + formattedEntries.add("--" + longOpt); + } else { + formattedEntries.add(trimmed); + } + continue; + } + + formattedEntries.add(trimmed); + } + + return String.join(", ", formattedEntries); + } + + private String resolveHealthReportLongOption(String shortOpt) { + switch (shortOpt) { + case "h": + return "help"; + case "e": + return "email"; + case "c": + return "check"; + case "f": + return "for"; + case "r": + return "report"; + default: + return null; + } + } + + private String appendCheckName(String value) { + try { + int checkIndex = Integer.parseInt(value.trim()); + String checkName = HealthReport.getCheckName(checkIndex); + if (checkName != null) { + return checkIndex + " (" + checkName + ")"; + } + return value; + } catch (NumberFormatException e) { + return value; + } + } + /** * Compare two reports based on the specified `from` and `to` dates. * If the reports are not found, log an appropriate message. * If the reports are found, generate a comparison report showing the differences. + * The comparison is based on the intersection of check names present in both reports. * * @param context the application context */ @@ -371,16 +549,15 @@ private void compareReports(Context context) { try { context.setCurrentUser(ePersonService.find(context, getEpersonIdentifier())); - ReportResult fromReport = specificCheck != -1 - ? reportResultService.findByLastModifiedAndCheckType(context, from, specificCheck) - : reportResultService.findByLastModified(context, from); + ReportResult fromReport = reportResultService.find(context, sourceReportId); + ReportResult toReport = reportResultService.find(context, targetReportId); - ReportResult toReport = specificCheck != -1 - ? reportResultService.findByLastModifiedAndCheckType(context, to, specificCheck) - : reportResultService.findByLastModified(context, to); - - if (fromReport == null || toReport == null) { - handler.logInfo("No reports found for specified dates."); + if (fromReport == null) { + handler.logInfo("No report found for report ID: " + sourceReportId); + return; + } + if (toReport == null) { + handler.logInfo("No report found for report ID: " + targetReportId); return; } @@ -406,9 +583,132 @@ private void compareReports(Context context) { } } + /** + * Holds the result of normalizing two reports to their intersection, + * including information about checks that were skipped (present in one report only). + */ + private static class NormalizationResult { + final String normalizedFromJson; + final String normalizedToJson; + /** Check names that exist only in the "from" report. */ + final List onlyInFrom; + /** Check names that exist only in the "to" report. */ + final List onlyInTo; + /** True when the two reports share at least one check eligible for comparison. */ + final boolean hasCommonChecks; + + NormalizationResult(String normalizedFromJson, String normalizedToJson, + List onlyInFrom, List onlyInTo, + boolean hasCommonChecks) { + this.normalizedFromJson = normalizedFromJson; + this.normalizedToJson = normalizedToJson; + this.onlyInFrom = onlyInFrom; + this.onlyInTo = onlyInTo; + this.hasCommonChecks = hasCommonChecks; + } + } + + /** + * Normalize two report JSON strings so that they only contain checks + * that are present (by name) in both reports. This allows correct comparison + * when reports were created with different check selections. + * + * If the `-c` option was specified, additionally filters to only include + * checks matching the specified check index (by name from the configured check list). + * + * @param fromJson the JSON string of the "from" report + * @param toJson the JSON string of the "to" report + * @return a {@link NormalizationResult} containing normalized JSON and skipped check info + * @throws IOException if JSON parsing fails + */ + private NormalizationResult normalizeReportsToIntersection(String fromJson, String toJson) throws IOException { + JsonNode fromRoot = mapper.readTree(fromJson); + JsonNode toRoot = mapper.readTree(toJson); + + JsonNode fromChecks = fromRoot.get("checks"); + JsonNode toChecks = toRoot.get("checks"); + + if (fromChecks == null || toChecks == null || !fromChecks.isArray() || !toChecks.isArray()) { + return new NormalizationResult(fromJson, toJson, + new ArrayList<>(), new ArrayList<>(), true); + } + + // Build maps of check name -> check node for both reports + Map fromCheckMap = new LinkedHashMap<>(); + for (JsonNode check : fromChecks) { + JsonNode nameNode = check.get("name"); + if (nameNode != null) { + fromCheckMap.put(nameNode.asText(), check); + } + } + + Map toCheckMap = new LinkedHashMap<>(); + for (JsonNode check : toChecks) { + JsonNode nameNode = check.get("name"); + if (nameNode != null) { + toCheckMap.put(nameNode.asText(), check); + } + } + + // Compute intersection of check names + List commonNames = new ArrayList<>(fromCheckMap.keySet()); + commonNames.retainAll(toCheckMap.keySet()); + + // If specificChecks are set, further filter to only those check names + if (!specificChecks.isEmpty()) { + List targetCheckNames = new ArrayList<>(); + for (int checkIndex : specificChecks) { + String targetCheckName = HealthReport.getCheckName(checkIndex); + if (targetCheckName != null) { + targetCheckNames.add(targetCheckName); + } + } + commonNames.retainAll(targetCheckNames); + } + + if (commonNames.isEmpty()) { + handler.logInfo("No common checks found between the two reports for comparison."); + } + + // Determine checks that are only in one report + List onlyInFrom = new ArrayList<>(fromCheckMap.keySet()); + onlyInFrom.removeAll(toCheckMap.keySet()); + List onlyInTo = new ArrayList<>(toCheckMap.keySet()); + onlyInTo.removeAll(fromCheckMap.keySet()); + + // When specific checks are requested, do not report other checks as skipped + if (!specificChecks.isEmpty()) { + onlyInFrom.clear(); + onlyInTo.clear(); + } + + // Build normalized JSON with only the common checks (in the same order) + ObjectNode normalizedFrom = mapper.createObjectNode(); + ArrayNode normalizedFromChecks = mapper.createArrayNode(); + for (String name : commonNames) { + normalizedFromChecks.add(fromCheckMap.get(name)); + } + normalizedFrom.set("checks", normalizedFromChecks); + + ObjectNode normalizedTo = mapper.createObjectNode(); + ArrayNode normalizedToChecks = mapper.createArrayNode(); + for (String name : commonNames) { + normalizedToChecks.add(toCheckMap.get(name)); + } + normalizedTo.set("checks", normalizedToChecks); + + return new NormalizationResult( + mapper.writeValueAsString(normalizedFrom), + mapper.writeValueAsString(normalizedTo), + onlyInFrom, onlyInTo, + !commonNames.isEmpty()); + } + /** * Generate a comparison report between two ReportResult objects. * The report includes the type, last modified dates, and the differences in JSON format. + * When comparing reports with different check selections, only the intersection + * of common check names is compared. * * @param fromReport the "from" report * @param toReport the "to" report @@ -423,6 +723,11 @@ private String generateReportComparison(ReportResult fromReport, ReportResult to return "One of the reports has no value. Cannot compare."; } + // Normalize both reports to contain only intersection of check names + NormalizationResult normalized = normalizeReportsToIntersection(fromJson, toJson); + String normalizedFromJson = normalized.normalizedFromJson; + String normalizedToJson = normalized.normalizedToJson; + StringBuilder sb = new StringBuilder(); // Header @@ -436,29 +741,93 @@ private String generateReportComparison(ReportResult fromReport, ReportResult to // Report metadata sb.append("Report Type: ").append(toReport.getType()).append("\n"); - sb.append("From: ").append(fromReport.getLastModified()).append("\n"); - sb.append("To: ").append(toReport.getLastModified()).append("\n"); + sb.append("Source Report: ID ").append(fromReport.getID()) + .append(" at ").append(fromReport.getLastModified()).append("\n"); + sb.append("Target Report: ID ").append(toReport.getID()) + .append(" at ").append(toReport.getLastModified()).append("\n"); // Calculate time period String timePeriod = calculateTimePeriod(fromReport.getLastModified(), toReport.getLastModified()); sb.append("Report Period: ").append(timePeriod).append("\n\n"); + // When there are no checks in common between the two reports there is nothing to diff. + // In that case, only show the executive summary and the list of skipped checks so the + // user can immediately see why the comparison was not performed. + if (!normalized.hasCommonChecks) { + appendSkippedChecksSection(sb, normalized, fromReport, toReport); + return sb.toString(); + } + // Enhanced Key Changes Table - String keyChangesTable = generateEnhancedKeyChangesTable(fromJson, toJson, - fromReport.getLastModified(), toReport.getLastModified()); + String keyChangesTable = generateEnhancedKeyChangesTable(normalizedFromJson, normalizedToJson, + fromReport.getID(), toReport.getID()); sb.append(keyChangesTable); - // Detailed Change Log - sb.append("Section 2: Detailed Change Log\n\n"); + // Keep output concise when the compared (common) checks are identical. + if (!hasDifferences(normalizedFromJson, normalizedToJson)) { + return sb.toString(); + } + + // Section 2: Skipped Checks (not present in both reports) + appendSkippedChecksSection(sb, normalized, fromReport, toReport); + + // Section 3: Detailed Change Log + sb.append("Section 3: Detailed Change Log\n\n"); sb.append("Changes Summary\n"); - String detailedSummary = generateDetailedSummary(fromJson, toJson); + String detailedSummary = generateDetailedSummary(normalizedFromJson, normalizedToJson); sb.append(detailedSummary).append("\n"); - sb.append(generateDiff(fromJson, toJson)); + sb.append(generateDiff(normalizedFromJson, normalizedToJson)); return sb.toString(); } + /** + * Append the "Section 2: Skipped Checks" block listing checks that are present + * in only one of the compared reports. Appends nothing when no check was skipped. + * + * @param sb the StringBuilder to append to + * @param normalized the normalization result holding the skipped check names + * @param fromReport the "from" report + * @param toReport the "to" report + */ + private void appendSkippedChecksSection(StringBuilder sb, NormalizationResult normalized, + ReportResult fromReport, ReportResult toReport) { + if (normalized.onlyInFrom.isEmpty() && normalized.onlyInTo.isEmpty()) { + return; + } + sb.append("Section 2: Skipped Checks\n\n"); + sb.append("The following checks could not be compared because they were not present in " + + "both reports.\n\n"); + if (!normalized.onlyInFrom.isEmpty()) { + sb.append("Only in source report (ID ").append(fromReport.getID()).append("):\n"); + for (String name : normalized.onlyInFrom) { + sb.append(" - ").append(name).append("\n"); + } + sb.append("\n"); + } + if (!normalized.onlyInTo.isEmpty()) { + sb.append("Only in target report (ID ").append(toReport.getID()).append("):\n"); + for (String name : normalized.onlyInTo) { + sb.append(" - ").append(name).append("\n"); + } + sb.append("\n"); + } + } + + /** + * Determine whether two normalized report JSON payloads differ. + * + * @param oldJson source report JSON + * @param newJson target report JSON + * @return true if there is at least one JSON Patch operation, false otherwise + * @throws IOException if JSON parsing fails + */ + private boolean hasDifferences(String oldJson, String newJson) throws IOException { + JsonNode patch = JsonDiff.asJson(mapper.readTree(oldJson), mapper.readTree(newJson)); + return patch.isArray() && !patch.isEmpty(); + } + /** * Calculate the time period between two dates with human-readable format. * @@ -483,6 +852,18 @@ private String padRight(String text, int width) { java.util.Objects.toString(text, "")); } + /** + * Check whether a JSON node carries no usable value, i.e. it is {@code null}, + * a missing node or a JSON null. Note this is different from {@link JsonNode#isEmpty()}, + * which checks for empty containers. + * + * @param node the JSON node to check, may be null + * @return true if the node is null, missing or a JSON null + */ + private static boolean isNullOrMissing(JsonNode node) { + return node == null || node.isMissingNode() || node.isNull(); + } + /** * Get a display-friendly version of a JSON node value. * @@ -490,7 +871,7 @@ private String padRight(String text, int width) { * @return display string */ private String getDisplayValue(JsonNode node) { - if (node == null || node.isMissingNode() || node.isNull()) { + if (isNullOrMissing(node)) { return "null"; } @@ -513,6 +894,18 @@ private String getDisplayValue(JsonNode node) { * @return difference string */ private String calculateDifference(JsonNode oldValue, JsonNode newValue) { + boolean oldMissing = isNullOrMissing(oldValue); + boolean newMissing = isNullOrMissing(newValue); + if (oldMissing || newMissing) { + if (oldMissing && !newMissing) { + return "Added"; + } + if (newMissing && !oldMissing) { + return "Removed"; + } + return "Changed"; + } + if (oldValue.isNumber() && newValue.isNumber()) { long oldNum = oldValue.asLong(); long newNum = newValue.asLong(); @@ -589,18 +982,197 @@ private String formatBytes(long bytes) { return (bytes / (1024 * 1024 * 1024)) + " GB"; } + /** + * Format an unsigned byte count with two-decimal precision for KB/MB/GB; used for value + * columns of byte-typed fields in the Key Changes table. + * + * @param bytes byte count (negatives are treated as their absolute value) + * @return formatted string, e.g. {@code 65.32 MB} + */ + private String formatBytesHuman(long bytes) { + long abs = Math.abs(bytes); + if (abs < 1024L) { + return abs + " B"; + } + if (abs < 1024L * 1024) { + return String.format(java.util.Locale.ROOT, "%.2f KB", abs / 1024.0); + } + if (abs < 1024L * 1024 * 1024) { + return String.format(java.util.Locale.ROOT, "%.2f MB", abs / (1024.0 * 1024)); + } + return String.format(java.util.Locale.ROOT, "%.2f GB", abs / (1024.0 * 1024 * 1024)); + } + + /** + * Format a (possibly negative) byte delta into a signed, human-readable string used in the + * Difference column for byte-typed fields. Produces values such as {@code -5.71 KB} or + * {@code +123 B} so administrators don't have to read raw byte counts. + * + * @param bytes signed byte delta + * @return formatted signed string + */ + private String formatSignedBytesHuman(long bytes) { + if (bytes == 0) { + return "0 B"; + } + String sign = bytes > 0 ? "+" : "-"; + long abs = Math.abs(bytes); + if (abs < 1024L) { + return sign + abs + " B"; + } + if (abs < 1024L * 1024) { + return String.format(java.util.Locale.ROOT, "%s%.2f KB", sign, abs / 1024.0); + } + if (abs < 1024L * 1024 * 1024) { + return String.format(java.util.Locale.ROOT, "%s%.2f MB", sign, abs / (1024.0 * 1024)); + } + return String.format(java.util.Locale.ROOT, "%s%.2f GB", sign, abs / (1024.0 * 1024 * 1024)); + } + + /** + * Resolve a field path with attribute selectors to a JSON value. + *

+ * Supports XPath-like selector syntax for matching array elements by a named field: + * 

+     *   /checks/[name=General Information]/report/publishedItems
+     *
+ * The segment {@code [name=General Information]} means: find the element in the {@code checks} + * array whose {@code "name"} field equals {@code "General Information"}. + *

+ * Regular path segments (e.g. {@code /report/collectionsSizesInfo/totalSize}) are resolved + * as standard JSON object field traversal. Numeric segments (e.g. {@code /0}) are resolved + * as array indices. + * + * @param rootNode the root JSON node to resolve against + * @param fieldPath the selector path, e.g. + * {@code /checks/[name=Item summary]/report/communitiesCount} + * @return the resolved {@link JsonNode}, or {@code null} if not found + */ + private JsonNode resolveFieldPath(JsonNode rootNode, String fieldPath) { + if (fieldPath == null || rootNode == null) { + return null; + } + + // Remove leading slash and split into segments + String path = fieldPath.startsWith("/") ? fieldPath.substring(1) : fieldPath; + // Split carefully: we need to handle segments like [name=General Information] + // which contain spaces but no slashes + List segments = splitPathSegments(path); + + JsonNode current = rootNode; + for (String segment : segments) { + if (current == null) { + return null; + } + + if (segment.startsWith("[") && segment.endsWith("]")) { + // Attribute selector, e.g. [name=General Information] + // The previous segment should have navigated us to an array node + if (!current.isArray()) { + return null; + } + String selectorContent = segment.substring(1, segment.length() - 1); + int eqIndex = selectorContent.indexOf('='); + if (eqIndex < 0) { + return null; + } + String attrName = selectorContent.substring(0, eqIndex).trim(); + String attrValue = selectorContent.substring(eqIndex + 1).trim(); + + // Find matching element in the array + JsonNode matched = null; + for (JsonNode element : current) { + JsonNode attrNode = element.get(attrName); + if (attrNode != null && attrValue.equals(attrNode.asText())) { + matched = element; + break; + } + } + current = matched; + } else if (current.isArray() && segment.matches("\\d+")) { + // Numeric index into array + int index = Integer.parseInt(segment); + current = (index >= 0 && index < current.size()) ? current.get(index) : null; + } else { + // Regular object field + current = current.get(segment); + } + } + + return current; + } + + /** + * Split a path string into segments, keeping bracket selectors as single segments. + * For example, {@code "checks/[name=General Information]/report/directoryStats/0/size_bytes"} + * becomes: {@code ["checks", "[name=General Information]", "report", "directoryStats", "0", "size_bytes"]}. + * + *

Limitation: The parser finds the first {@code ]} after an opening {@code [}, so check + * names that themselves contain bracket characters (e.g., {@code [name=Check [beta]]}) are not + * supported and will produce incorrect segments. Check names must not contain {@code [} or {@code ]}. + * + * @param path the path to split (without leading slash) + * @return list of path segments + */ + private List splitPathSegments(String path) { + List segments = new ArrayList<>(); + int i = 0; + while (i < path.length()) { + if (path.charAt(i) == '[') { + // Find matching closing bracket + int closeBracket = path.indexOf(']', i); + if (closeBracket < 0) { + closeBracket = path.length() - 1; + } + segments.add(path.substring(i, closeBracket + 1)); + i = closeBracket + 1; + // Skip following slash if present + if (i < path.length() && path.charAt(i) == '/') { + i++; + } + } else { + // Regular segment - find next slash or bracket + int nextSlash = path.indexOf('/', i); + int nextBracket = path.indexOf('[', i); + int end; + if (nextSlash < 0 && nextBracket < 0) { + end = path.length(); + } else if (nextSlash < 0) { + end = nextBracket; + } else if (nextBracket < 0) { + end = nextSlash; + } else { + end = Math.min(nextSlash, nextBracket); + } + String segment = path.substring(i, end); + if (!segment.isEmpty()) { + segments.add(segment); + } + i = end; + // Skip slash separator + if (i < path.length() && path.charAt(i) == '/') { + i++; + } + } + } + return segments; + } + /** * Generate enhanced key changes table with dynamic sizing and configurable field names. + * Uses selector-based field resolution that works regardless of check ordering or selection. + * Field paths use XPath-like syntax, e.g. {@code /checks/[name=Item summary]/report/publishedItems}. * * @param oldJson the old JSON report * @param newJson the new JSON report - * @param fromDate the date of the old report - * @param toDate the date of the new report + * @param sourceReportId the ID of the source (older) report, used in column headers + * @param targetReportId the ID of the target (newer) report, used in column headers * @return formatted table string * @throws IOException if JSON parsing fails */ private String generateEnhancedKeyChangesTable(String oldJson, String newJson, - Date fromDate, Date toDate) throws IOException { + Integer sourceReportId, Integer targetReportId) + throws IOException { loadFieldConfiguration(); JsonNode oldNode = mapper.readTree(oldJson); @@ -610,15 +1182,37 @@ private String generateEnhancedKeyChangesTable(String oldJson, String newJson, List changes = new ArrayList<>(); for (String fieldPath : fieldOrder) { - JsonNode oldValue = getValueFromPath(oldNode, fieldPath); - JsonNode newValue = getValueFromPath(newNode, fieldPath); + JsonNode oldValue = resolveFieldPath(oldNode, fieldPath); + JsonNode newValue = resolveFieldPath(newNode, fieldPath); + + // Skip fields that don't exist in either report (check not present in both) + boolean oldMissing = oldValue == null || oldValue.isMissingNode(); + boolean newMissing = newValue == null || newValue.isMissingNode(); + if (oldMissing && newMissing) { + continue; + } - if (!Objects.equals(getDisplayValue(oldValue), getDisplayValue(newValue))) { - String displayName = fieldMappings.getOrDefault(fieldPath, fieldPath); - String oldDisplay = getDisplayValue(oldValue); - String newDisplay = getDisplayValue(newValue); - String difference = calculateDifference(oldValue, newValue); + // For byte-typed fields (paths ending with size_bytes) render values and diff in + // human-readable units so administrators get e.g. -5.71 KB instead of -5850. + boolean isByteField = fieldPath.endsWith("size_bytes"); + String oldDisplay; + String newDisplay; + String difference; + if (isByteField && oldValue != null && newValue != null + && oldValue.isNumber() && newValue.isNumber()) { + long oldBytes = oldValue.asLong(); + long newBytes = newValue.asLong(); + oldDisplay = formatBytesHuman(oldBytes); + newDisplay = formatBytesHuman(newBytes); + difference = formatSignedBytesHuman(newBytes - oldBytes); + } else { + oldDisplay = getDisplayValue(oldValue); + newDisplay = getDisplayValue(newValue); + difference = calculateDifference(oldValue, newValue); + } + if (!Objects.equals(oldDisplay, newDisplay)) { + String displayName = fieldMappings.getOrDefault(fieldPath, fieldPath); changes.add(new TableRow(displayName, oldDisplay, newDisplay, difference)); } } @@ -628,16 +1222,16 @@ private String generateEnhancedKeyChangesTable(String oldJson, String newJson, "No significant changes detected between reports.\n\n"; } - // Format dates for column headers using thread-safe DateTimeFormatter - String fromDateStr = fromDate.toInstant().atZone(java.time.ZoneId.systemDefault()).format(FORMATTER); - String toDateStr = toDate.toInstant().atZone(java.time.ZoneId.systemDefault()).format(FORMATTER); + // Compact ID-only column headers; full timestamps appear in the Executive Summary above. + String fromHeader = "Source: ID " + sourceReportId; + String toHeader = "Target: ID " + targetReportId; // Calculate dynamic column widths including header content int fieldWidth = Math.max("Field".length(), changes.stream().mapToInt(r -> r.field.length()).max().orElse(25)); - int oldWidth = Math.max(fromDateStr.length(), + int oldWidth = Math.max(fromHeader.length(), changes.stream().mapToInt(r -> r.oldValue.length()).max().orElse(15)); - int newWidth = Math.max(toDateStr.length(), + int newWidth = Math.max(toHeader.length(), changes.stream().mapToInt(r -> r.newValue.length()).max().orElse(15)); int diffWidth = Math.max("Difference".length(), changes.stream().mapToInt(r -> r.difference.length()).max().orElse(12)); @@ -652,8 +1246,8 @@ private String generateEnhancedKeyChangesTable(String oldJson, String newJson, // Header with separator table.append(separator).append("\n"); table.append("| ").append(padRight("Field", fieldWidth)) - .append(" | ").append(padRight(fromDateStr, oldWidth)) - .append(" | ").append(padRight(toDateStr, newWidth)) + .append(" | ").append(padRight(fromHeader, oldWidth)) + .append(" | ").append(padRight(toHeader, newWidth)) .append(" | ").append(padRight("Difference", diffWidth)) .append(" |\n"); table.append(separator).append("\n"); @@ -672,43 +1266,6 @@ private String generateEnhancedKeyChangesTable(String oldJson, String newJson, return table.toString(); } - /** - * Get value from JSON node using path notation (JSON Pointer style). - */ - private JsonNode getValueFromPath(JsonNode node, String path) { - try { - // Use Jackson's JSON Pointer functionality for paths like /checks/0/report/publishedItems - if (path.startsWith("/")) { - return node.at(path); - } - - // Fallback for simple dot notation paths - return getValueFromSimplePath(node, path); - } catch (Exception e) { - return null; - } - } - - /** - * Get value from simple dot-notation path. - */ - private JsonNode getValueFromSimplePath(JsonNode node, String path) { - if (path.isEmpty()) { - return node; - } - String[] parts = path.split("\\."); - JsonNode current = node; - - for (String part : parts) { - if (current == null || !current.has(part)) { - return null; - } - current = current.get(part); - } - - return current; - } - /** * Simple data class for table rows. */ @@ -788,10 +1345,12 @@ private String generateDetailedSummary(String oldJson, String newJson) throws IO public void printHelp() { handler.printHelp(getScriptConfiguration().getOptions(), getScriptConfiguration().getName()); handler.logInfo("This script compares two health reports and shows the differences between them."); - handler.logInfo("You can specify the 'from' and 'to' dates to compare reports from specific dates."); - handler.logInfo("If you want to see all available report dates, use the '-d' option."); + handler.logInfo("Use '-s/--source' and '-t/--target' with report IDs to pick the source" + + " and target report."); + handler.logInfo("Use '-l/--list' to list all available reports with their IDs and timestamps."); + handler.logInfo("Use '-m/--max' together with '--list' to limit how many entries are shown."); handler.logInfo("If you want to compare a specific check, use the '-c' option with the check index, " + - "in this case you must also specify the `from` and `to` dates."); + "in this case you must also specify the source and target report IDs."); handler.logInfo("If you want to send the report to a specified email address, use '-e'."); } @@ -811,7 +1370,7 @@ public static String generateDiff(String oldJson, String newJson) throws IOExcep JsonNode patch = JsonDiff.asJson(oldNode, newNode); - if (!patch.isArray() || patch.size() == 0) { + if (!patch.isArray() || patch.isEmpty()) { return "No differences found."; } @@ -950,14 +1509,11 @@ private static void appendTest(StringBuilder sb, JsonNode op, String path) { /** * Return the node’s JSON-string representation, so that special characters * like newline (\n) appear as "\\n" inside the returned quote marks. + * For any primitive or object/array, toString() returns valid JSON. MissingNode.toString() + * would return an empty string, so null/missing/JSON-null nodes are all rendered as "null". */ private static String nodeToEscapedString(JsonNode node) { - if (node == null || node.isMissingNode() || node.isNull()) { - return "null"; - } - // For any primitive or object/array, toString() returns valid JSON. - // In particular, a text node will come out as "\"some text\\n\"" (with \\n escaped). - return node.toString(); + return isNullOrMissing(node) ? "null" : node.toString(); } } @@ -966,14 +1522,20 @@ private static String nodeToEscapedString(JsonNode node) { * Used for displaying report dates with their arguments. */ class DateWithArgs { + private final Integer id; private final String date; private final String args; - public DateWithArgs(String date, String args) { + public DateWithArgs(Integer id, String date, String args) { + this.id = id; this.date = date; this.args = args; } + public Integer getId() { + return id; + } + public String getDate() { return date; } diff --git a/dspace-api/src/main/java/org/dspace/app/reportdiff/ReportDiffScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/app/reportdiff/ReportDiffScriptConfiguration.java index da1933590bb2..5067b0496a5a 100644 --- a/dspace-api/src/main/java/org/dspace/app/reportdiff/ReportDiffScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/app/reportdiff/ReportDiffScriptConfiguration.java @@ -7,6 +7,7 @@ */ package org.dspace.app.reportdiff; +import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.dspace.app.healthreport.HealthReport; import org.dspace.scripts.configuration.ScriptConfiguration; @@ -33,26 +34,32 @@ public void setDspaceRunnableClass(Class dspaceRunnableClass) { public Options getOptions() { if (options == null) { Options options = new Options(); - options.addOption("i", "info", false, + options.addOption("h", "help", false, "Show help information."); options.addOption("e", "email", true, "Send report to this email address."); options.getOption("e").setType(String.class); - options.addOption("c", "check", true, - String.format("Perform only specific check (use index from 0 to %d, " + - "otherwise perform default checks).", HealthReport.getNumberOfChecks() - 1)); - options.getOption("c").setType(String.class); + Option checkOption = Option.builder("c").longOpt("check").hasArgs() + .desc(String.format("Filter comparison to one or more specific checks by index (0 to %d). " + + "Repeat the flag (e.g. -c 1 -c 3) to compare multiple checks from both reports.", + HealthReport.getNumberOfChecks() - 1)) + .type(String.class) + .build(); + options.addOption(checkOption); - options.addOption("d", "dates", false, "Show all report dates"); + options.addOption("l", "list", false, + "List available reports (ID, timestamp, args). Use to find report IDs."); - options.addOption("l", "limit", true, - "Limit the number of entries (use only with -d). If omitted, all entries are shown."); - options.getOption("l").setType(String.class); + options.addOption("m", "max", true, + "Limit the number of entries (use only with -l). If omitted, all entries are shown."); + options.getOption("m").setType(String.class); - options.addOption("f", "from", true,"Report from specific date [YYYY-MM-DD HH:mm:ss.SSS]."); - options.getOption("f").setType(String.class); + options.addOption("s", "source", true, + "Source report ID to compare from."); + options.getOption("s").setType(String.class); - options.addOption("t", "to", true,"Report to specific date [YYYY-MM-DD HH:mm:ss.SSS]."); + options.addOption("t", "target", true, + "Target report ID to compare against."); options.getOption("t").setType(String.class); super.options = options; diff --git a/dspace-api/src/main/java/org/dspace/content/ReportResultServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/ReportResultServiceImpl.java index 4bfa18b668ba..15722dfee6c1 100644 --- a/dspace-api/src/main/java/org/dspace/content/ReportResultServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/ReportResultServiceImpl.java @@ -52,12 +52,6 @@ public ReportResult findByLastModified(Context context, Date lastModified) throw return reportResultDAO.findByLastModified(context, lastModified); } - @Override - public ReportResult findByLastModifiedAndCheckType(Context context, Date lastModified, int checkType) - throws SQLException { - return reportResultDAO.findByLastModifiedAndCheckType(context, lastModified, checkType); - } - @Override public void delete(Context context, ReportResult reportResult) throws SQLException { reportResultDAO.delete(context, reportResult); diff --git a/dspace-api/src/main/java/org/dspace/content/dao/ReportResultDAO.java b/dspace-api/src/main/java/org/dspace/content/dao/ReportResultDAO.java index 8f2aeabb2e7c..6cf00a632728 100644 --- a/dspace-api/src/main/java/org/dspace/content/dao/ReportResultDAO.java +++ b/dspace-api/src/main/java/org/dspace/content/dao/ReportResultDAO.java @@ -30,16 +30,5 @@ public interface ReportResultDAO extends GenericDAO { * @throws SQLException if a database error occurs */ ReportResult findByLastModified(Context context, Date lastModified) throws SQLException; - - /** - * Find a ReportResult by its last modified date and check type. - * - * @param context the DSpace context - * @param lastModified the exact last modified date to search for - * @param checkType the check type index to filter by (searches within args field) - * @return the ReportResult matching both criteria, or null if not found - * @throws SQLException if a database error occurs - */ - ReportResult findByLastModifiedAndCheckType(Context context, Date lastModified, int checkType) throws SQLException; } diff --git a/dspace-api/src/main/java/org/dspace/content/dao/impl/ReportResultDAOImpl.java b/dspace-api/src/main/java/org/dspace/content/dao/impl/ReportResultDAOImpl.java index a85d53f5f4be..f8706565da76 100644 --- a/dspace-api/src/main/java/org/dspace/content/dao/impl/ReportResultDAOImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/dao/impl/ReportResultDAOImpl.java @@ -34,18 +34,4 @@ public ReportResult findByLastModified(Context context, Date lastModified) throw return singleResult(query); } - - @Override - public ReportResult findByLastModifiedAndCheckType(Context context, Date lastModified, int checkType) - throws SQLException { - // Use string matching for checkType in args (args contains command line options like "-c: 0") - Query query = createQuery(context, "SELECT r FROM ReportResult r WHERE r.lastModified = :lastModified " + - "AND r.args LIKE :argsPattern"); - - query.setParameter("lastModified", lastModified); - query.setParameter("argsPattern", "%-c: " + checkType + "%"); - query.setHint("org.hibernate.cacheable", Boolean.TRUE); - - return singleResult(query); - } } diff --git a/dspace-api/src/main/java/org/dspace/content/service/ReportResultService.java b/dspace-api/src/main/java/org/dspace/content/service/ReportResultService.java index aa52ec31de99..fabfaeb997a7 100644 --- a/dspace-api/src/main/java/org/dspace/content/service/ReportResultService.java +++ b/dspace-api/src/main/java/org/dspace/content/service/ReportResultService.java @@ -70,16 +70,6 @@ public interface ReportResultService { */ ReportResult findByLastModified(Context context, Date lastModified) throws SQLException; - /** - * Find a ReportResult by last modified date and check type. - * - * @param context the DSpace context - * @param lastModified the exact last modified date to search for - * @param checkType the check type index to filter by - * @return the matching ReportResult, or null if not found - * @throws SQLException if a database error occurs - */ - ReportResult findByLastModifiedAndCheckType(Context context, Date lastModified, int checkType) throws SQLException; /** * Deletes the specified ReportResult instance in the given context. diff --git a/dspace-api/src/main/java/org/dspace/health/Report.java b/dspace-api/src/main/java/org/dspace/health/Report.java deleted file mode 100644 index ebb2ffd688c0..000000000000 --- a/dspace-api/src/main/java/org/dspace/health/Report.java +++ /dev/null @@ -1,229 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.health; - -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map.Entry; -import java.util.StringTokenizer; -import javax.mail.MessagingException; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.dspace.core.Email; -import org.dspace.core.factory.CoreServiceFactory; -import org.dspace.core.service.PluginService; -import org.dspace.services.ConfigurationService; -import org.dspace.services.factory.DSpaceServicesFactory; - -/** - * @author LINDAT/CLARIN dev team - */ -public class Report { - - private static final Logger log = LogManager.getLogger(Report.class); - public static final String EMAIL_PATH = "config/emails/healthcheck"; - // store the individual check reports - private final StringBuilder summary_; - - // ctor - // - public Report() { - summary_ = new StringBuilder(); - } - - // run checks - // - public void run(List to_perform, ReportInfo ri) { - - int pos = -1; - for (Entry check_entry : checks().entrySet()) { - ++pos; - if (null != to_perform && !to_perform.contains(pos)) { - continue; - } - String check_name = check_entry.getKey(); - Check check = check_entry.getValue(); - - log.info(String.format("#%d. Processing [%s] at [%s]", - pos, check_name, new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss.SSS").format(new Date()))); - - try { - // do the stuff - check.report(ri); - store(check_name, check.took_, check.report_); - - } catch (Exception e) { - store( - check_name, - -1, - "Exception occurred when processing report - " + ExceptionUtils.getStackTrace(e) - ); - } - } - } - - // create check list - public static LinkedHashMap checks() { - LinkedHashMap checks = new LinkedHashMap<>(); - String check_names[] = DSpaceServicesFactory.getInstance().getConfigurationService() - .getArrayProperty("healthcheck.checks"); - PluginService pluginService = CoreServiceFactory.getInstance().getPluginService(); - for (String check_name : check_names) { - Check check = (Check) pluginService.getNamedPlugin( - Check.class, check_name); - if (null != check) { - checks.put(check_name, check); - } else { - log.warn(String.format( - "Could not find implementation for [%s]", check_name)); - } - } - return checks; - } - - @Override - public String toString() { - return summary_.toString(); - } - - // - private void store(String name, long took, String report) { - name += String.format(" [took: %ds] [# lines: %d]", - took / 1000, - new StringTokenizer(report, "\r\n").countTokens() - ); - - String one_summary = String.format( - "\n#### %s\n%s\n\n###############################\n", - name, - report.replaceAll("\\s+$", "") - ); - summary_.append(one_summary); - - // output it - System.out.println(one_summary); - - } - - // main - // - - public static void main(String[] args) { - log.info("Starting healthcheck report..."); - - final String option_help = "h"; - final String option_email = "e"; - final String option_check = "c"; - final String option_last_n = "f"; - final String option_verbose = "v"; - - // command line options - Options options = new Options(); - options.addOption(option_help, "help", false, - "Show available checks and their index."); - options.addOption(option_email, "email", true, - "Send report to this email address."); - options.addOption(option_check, "check", true, - "Perform only specific check (use index starting from 0)."); - options.addOption(option_last_n, "for", true, - "For last N days."); - options.addOption(option_verbose, "verbose", false, - "Verbose report."); - - CommandLine cmdline = null; - try { - cmdline = new DefaultParser().parse(options, args); - } catch (ParseException e) { - log.fatal("Invalid command line " + e.toString(), e); - System.exit(1); - } - - if (cmdline.hasOption(option_help)) { - String checks_summary = ""; - int pos = 0; - for (String check_name : checks().keySet()) { - checks_summary += String.format("%d. %s\n", pos++, check_name); - } - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("dspace healthcheck", options); - System.out.println("\nAvailable checks:\n" + checks_summary); - return; - } - - // what to perform - List to_perform = null; - if (null != cmdline.getOptionValues(option_check)) { - to_perform = new ArrayList<>(); - for (String s : cmdline.getOptionValues('c')) { - to_perform.add(Integer.valueOf(s)); - } - } - - try { - ConfigurationService configurationService - = DSpaceServicesFactory.getInstance().getConfigurationService(); - // last n days - int for_last_n_days = configurationService.getIntProperty( - "healthcheck.last_n_days"); - if (cmdline.hasOption(option_last_n)) { - for_last_n_days = Integer.getInteger( - cmdline.getOptionValue(option_last_n)); - } - ReportInfo ri = new ReportInfo(for_last_n_days); - if (cmdline.hasOption(option_verbose)) { - ri.verbose(true); - } - - // run report - Report r = new Report(); - r.run(to_perform, ri); - log.info("reports generated..."); - - // send/output the report - if (cmdline.hasOption(option_email)) { - String to = cmdline.getOptionValue(option_email); - if (!to.contains("@")) { - to = configurationService.getProperty(to); - } - try { - String dspace_dir = configurationService.getProperty("dspace.dir"); - String email_path = dspace_dir.endsWith("/") ? dspace_dir - : dspace_dir + "/"; - email_path += Report.EMAIL_PATH; - log.info(String.format( - "Looking for email template at [%s]", email_path)); - Email email = Email.getEmail(email_path); - email.addRecipient(to); - email.addArgument(r.toString()); - email.send(); - } catch (IOException | MessagingException e) { - log.fatal("Error sending email:", e); - System.err.println("Error sending email:\n" + e.getMessage()); - System.exit(1); - } - } - - } catch (Exception e) { - log.fatal(e); - e.printStackTrace(); - } - } - -} diff --git a/dspace-api/src/main/resources/report-diff-fields.json b/dspace-api/src/main/resources/report-diff-fields.json index a1e1f687aefc..23891359a95e 100644 --- a/dspace-api/src/main/resources/report-diff-fields.json +++ b/dspace-api/src/main/resources/report-diff-fields.json @@ -1,54 +1,54 @@ { "fieldMappings": { - "/checks/0/report/directoryStats/0/size_bytes": "Assetstore Size (bytes)", - "/checks/0/report/directoryStats/1/size_bytes": "Log Directory Size (bytes)", - "/checks/1/report/communitiesCount": "Communities", - "/checks/1/report/collectionsCount": "Collections", - "/checks/1/report/collectionsSizesInfo/totalSize": "Total Content Size", - "/checks/1/report/itemsCount": "Items", - "/checks/1/report/publishedItems": "Published Items", - "/checks/1/report/notPublishedItems": "Unpublished Items", - "/checks/1/report/withdrawnItems": "Withdrawn Items", - "/checks/1/report/workspaceItemsCount": "Workspace Items", - "/checks/1/report/waitingForApproval": "Workflow Items", - "/checks/1/report/bitstreamsCount": "Bitstreams", - "/checks/1/report/bundlesCount": "Bundles", - "/checks/1/report/collectionsSizesInfo/orphanBitstreamsCount": "Orphaned Bitstreams", - "/checks/1/report/collectionsSizesInfo/deletedBitstreams": "Deleted Bitstreams", - "/checks/1/report/metadataValuesCount": "Metadata Values", - "/checks/1/report/handlesCount": "Handles", - "/checks/1/report/ePersonsCount": "Users", - "/checks/1/report/groupsCount": "Groups", - "/checks/2/report/selfRegistered": "Self Registered Users", - "/checks/2/report/subscribers": "Subscribers", - "/checks/2/report/subscribedCollections": "Subscribed Collections", - "/checks/2/report/emptyGroups": "Empty Groups", - "/checks/3/report/licenses": "Licenses" + "/checks/[name=General Information]/report/directoryStats/0/size_bytes": "Assetstore Size", + "/checks/[name=General Information]/report/directoryStats/1/size_bytes": "Log Directory Size", + "/checks/[name=Item summary]/report/communitiesCount": "Communities", + "/checks/[name=Item summary]/report/collectionsCount": "Collections", + "/checks/[name=Item summary]/report/collectionsSizesInfo/totalSize": "Total Content Size", + "/checks/[name=Item summary]/report/itemsCount": "Items", + "/checks/[name=Item summary]/report/publishedItems": "Published Items", + "/checks/[name=Item summary]/report/notPublishedItems": "Unpublished Items", + "/checks/[name=Item summary]/report/withdrawnItems": "Withdrawn Items", + "/checks/[name=Item summary]/report/workspaceItemsCount": "Workspace Items", + "/checks/[name=Item summary]/report/waitingForApproval": "Workflow Items", + "/checks/[name=Item summary]/report/bitstreamsCount": "Bitstreams", + "/checks/[name=Item summary]/report/bundlesCount": "Bundles", + "/checks/[name=Item summary]/report/collectionsSizesInfo/orphanBitstreamsCount": "Orphaned Bitstreams", + "/checks/[name=Item summary]/report/collectionsSizesInfo/deletedBitstreams": "Deleted Bitstreams", + "/checks/[name=Item summary]/report/metadataValuesCount": "Metadata Values", + "/checks/[name=Item summary]/report/handlesCount": "Handles", + "/checks/[name=Item summary]/report/ePersonsCount": "Users", + "/checks/[name=Item summary]/report/groupsCount": "Groups", + "/checks/[name=User summary]/report/selfRegistered": "Self Registered Users", + "/checks/[name=User summary]/report/subscribers": "Subscribers", + "/checks/[name=User summary]/report/subscribedCollections": "Subscribed Collections", + "/checks/[name=User summary]/report/emptyGroups": "Empty Groups", + "/checks/[name=License summary]/report/licenses": "Licenses" }, "fieldOrder": [ - "/checks/0/report/directoryStats/0/size_bytes", - "/checks/0/report/directoryStats/1/size_bytes", - "/checks/1/report/communitiesCount", - "/checks/1/report/collectionsCount", - "/checks/1/report/collectionsSizesInfo/totalSize", - "/checks/1/report/itemsCount", - "/checks/1/report/publishedItems", - "/checks/1/report/notPublishedItems", - "/checks/1/report/withdrawnItems", - "/checks/1/report/workspaceItemsCount", - "/checks/1/report/waitingForApproval", - "/checks/1/report/bitstreamsCount", - "/checks/1/report/bundlesCount", - "/checks/1/report/collectionsSizesInfo/orphanBitstreamsCount", - "/checks/1/report/collectionsSizesInfo/deletedBitstreams", - "/checks/1/report/metadataValuesCount", - "/checks/1/report/handlesCount", - "/checks/1/report/ePersonsCount", - "/checks/1/report/groupsCount", - "/checks/2/report/selfRegistered", - "/checks/2/report/subscribers", - "/checks/2/report/subscribedCollections", - "/checks/2/report/emptyGroups", - "/checks/3/report/licenses" + "/checks/[name=General Information]/report/directoryStats/0/size_bytes", + "/checks/[name=General Information]/report/directoryStats/1/size_bytes", + "/checks/[name=Item summary]/report/communitiesCount", + "/checks/[name=Item summary]/report/collectionsCount", + "/checks/[name=Item summary]/report/collectionsSizesInfo/totalSize", + "/checks/[name=Item summary]/report/itemsCount", + "/checks/[name=Item summary]/report/publishedItems", + "/checks/[name=Item summary]/report/notPublishedItems", + "/checks/[name=Item summary]/report/withdrawnItems", + "/checks/[name=Item summary]/report/workspaceItemsCount", + "/checks/[name=Item summary]/report/waitingForApproval", + "/checks/[name=Item summary]/report/bitstreamsCount", + "/checks/[name=Item summary]/report/bundlesCount", + "/checks/[name=Item summary]/report/collectionsSizesInfo/orphanBitstreamsCount", + "/checks/[name=Item summary]/report/collectionsSizesInfo/deletedBitstreams", + "/checks/[name=Item summary]/report/metadataValuesCount", + "/checks/[name=Item summary]/report/handlesCount", + "/checks/[name=Item summary]/report/ePersonsCount", + "/checks/[name=Item summary]/report/groupsCount", + "/checks/[name=User summary]/report/selfRegistered", + "/checks/[name=User summary]/report/subscribers", + "/checks/[name=User summary]/report/subscribedCollections", + "/checks/[name=User summary]/report/emptyGroups", + "/checks/[name=License summary]/report/licenses" ] } \ No newline at end of file diff --git a/dspace-api/src/test/java/org/dspace/scripts/HealthReportIT.java b/dspace-api/src/test/java/org/dspace/scripts/HealthReportIT.java index e77a907ef731..68142e18fcc4 100644 --- a/dspace-api/src/test/java/org/dspace/scripts/HealthReportIT.java +++ b/dspace-api/src/test/java/org/dspace/scripts/HealthReportIT.java @@ -14,13 +14,16 @@ import static org.hamcrest.Matchers.hasSize; import java.io.ByteArrayInputStream; +import java.io.File; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.util.HashSet; import java.util.List; import java.util.Set; import org.dspace.AbstractIntegrationTestWithDatabase; +import org.dspace.app.healthreport.HealthReport; import org.dspace.app.launcher.ScriptLauncher; import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; import org.dspace.builder.CollectionBuilder; @@ -31,6 +34,7 @@ import org.dspace.content.Collection; import org.dspace.content.Community; import org.dspace.content.Item; +import org.dspace.content.ReportResult; import org.dspace.content.clarin.ClarinLicense; import org.dspace.content.clarin.ClarinLicenseLabel; import org.dspace.content.clarin.ClarinLicenseResourceMapping; @@ -38,6 +42,7 @@ import org.dspace.content.factory.ContentServiceFactory; import org.dspace.content.service.BitstreamService; import org.dspace.content.service.BundleService; +import org.dspace.content.service.ReportResultService; import org.dspace.content.service.clarin.ClarinLicenseLabelService; import org.dspace.content.service.clarin.ClarinLicenseResourceMappingService; import org.dspace.content.service.clarin.ClarinLicenseService; @@ -68,7 +73,7 @@ public void testDefaultHealthcheckRun() throws Exception { List messages = testDSpaceRunnableHandler.getInfoMessages(); assertThat(messages, hasSize(1)); - assertThat(messages, hasItem(containsString("HEALTH REPORT:"))); + assertThat(messages, hasItem(containsString("HEALTH REPORT "))); } @Test @@ -139,4 +144,134 @@ public void testLicenseCheck() throws Exception { assertThat(messages, hasItem(containsString("UUIDs of items without license bundle:"))); assertThat(messages, hasItem(containsString("PUB"))); } + + /** + * Verifies that -h/--help prints help text and does not run any checks. + * use -h instead of -i. + */ + @Test + public void testHelpOption() throws Exception { + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "health-report", "-h" }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + assertThat(handler.getErrorMessages(), empty()); + List messages = handler.getInfoMessages(); + assertThat(messages, hasItem(containsString("HELP"))); + assertThat(messages, hasItem(containsString("Available checks:"))); + } + + /** + * Verifies that multiple values for a single -c option run only the specified checks. + * Supports multiple check selection (e.g. -c 0 3). + */ + @Test + public void testMultipleChecks() throws Exception { + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + // Run only check 0 (General Information) and check 3 (License summary): space-separated + String[] args = new String[] { "health-report", "-c", "0", "3" }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + assertThat(handler.getErrorMessages(), empty()); + List messages = handler.getInfoMessages(); + assertThat(messages, hasItem(containsString("HEALTH REPORT "))); + assertThat(messages, hasItem(containsString("General Information"))); + assertThat(messages, hasItem(containsString("License summary"))); + // Item summary (check index 1) should NOT be present + boolean hasItemSummary = messages.stream().anyMatch(m -> m.contains("Item summary:")); + assertThat("Only selected checks should run", hasItemSummary, org.hamcrest.Matchers.is(false)); + } + + /** + * Verifies that an out-of-range -c value causes a script error. + */ + @Test + public void testInvalidCheckOutOfRange() throws Exception { + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + int maxCheck = HealthReport.getNumberOfChecks() - 1; + String[] args = new String[] { "health-report", "-c", String.valueOf(maxCheck + 1) }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + assertThat(handler.getErrorMessages(), + hasItem(containsString("Must be an integer from 0 to " + maxCheck))); + } + + /** + * Verifies that a non-integer -c value causes a script error. + */ + @Test + public void testInvalidCheckNonInteger() throws Exception { + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "health-report", "-c", "abc" }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + assertThat(handler.getErrorMessages(), + hasItem(containsString("It has to be an integer number from 0 to"))); + } + + /** + * Verifies that a non-positive -f value (zero) causes a script error. + * Validate -f must be positive integer (greater than 0). + */ + @Test + public void testInvalidForDaysZero() throws Exception { + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "health-report", "-f", "0" }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + assertThat(handler.getErrorMessages(), + hasItem(containsString("Must be a positive integer (greater than 0)"))); + } + + /** + * Verifies that a non-integer -f value causes a script error. + * Validate -f must be integer. + */ + @Test + public void testInvalidForDaysNonInteger() throws Exception { + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "health-report", "-f", "notanumber" }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + assertThat(handler.getErrorMessages(), + hasItem(containsString("Must be a positive integer"))); + } + + /** + * Verifies that -r/--report saves report output to the specified file. + * -o/--output renamed to -r/--report. + */ + @Test + public void testReportFileSaved() throws Exception { + File tempFile = File.createTempFile("health-report-test-", ".txt"); + tempFile.deleteOnExit(); + + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "health-report", "-r", tempFile.getAbsolutePath() }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + assertThat(handler.getErrorMessages(), empty()); + assertThat("Report file must exist after -r option", tempFile.exists(), org.hamcrest.Matchers.is(true)); + String content = Files.readString(tempFile.toPath()); + assertThat("Report file must contain health report header", content, containsString("HEALTH REPORT ")); + } + + @Test + public void testStoredArgsContainAllCheckOptions() throws Exception { + ReportResultService reportResultService = ContentServiceFactory.getInstance().getReportResultService(); + + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "health-report", "-c", "2", "-c", "3" }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + context.reloadEntity(eperson); + List allReports = reportResultService.findAll(context); + // findAll() does not guarantee ordering; sort by lastModified so the newest report is last. + allReports.sort(java.util.Comparator.comparing(ReportResult::getLastModified)); + ReportResult latest = allReports.get(allReports.size() - 1); + + assertThat(handler.getErrorMessages(), empty()); + assertThat(latest.getArgs(), containsString("-c: 2")); + assertThat(latest.getArgs(), containsString("-c: 3")); + } } \ No newline at end of file diff --git a/dspace-api/src/test/java/org/dspace/scripts/ReportDiffIT.java b/dspace-api/src/test/java/org/dspace/scripts/ReportDiffIT.java index ce3e8b168be8..fc69aab0f37e 100644 --- a/dspace-api/src/test/java/org/dspace/scripts/ReportDiffIT.java +++ b/dspace-api/src/test/java/org/dspace/scripts/ReportDiffIT.java @@ -20,7 +20,6 @@ import java.util.regex.Pattern; import org.dspace.AbstractIntegrationTestWithDatabase; -import org.dspace.app.healthreport.HealthReport; import org.dspace.app.launcher.ScriptLauncher; import org.dspace.app.scripts.handler.impl.TestDSpaceRunnableHandler; import org.dspace.content.ReportResult; @@ -111,7 +110,7 @@ private String formatDate(Date date) { @Test public void testHelpInformation() throws Exception { TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "report-diff", "-i" }; + String[] args = new String[] { "report-diff", "-h" }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); List infoMessages = handler.getInfoMessages(); @@ -126,6 +125,7 @@ public void testShowDates() throws Exception { ReportResult report1 = reportResultService.create(context); report1.setType("healthcheck"); report1.setValue("{\"checks\":[]}"); + report1.setArgs("-c: 0\n-c: 0\n-r: reportout.txt\n-f: 3\n"); reportResultService.update(context, report1); // Force commit and flush to ensure timestamp is set context.commit(); @@ -136,6 +136,7 @@ public void testShowDates() throws Exception { ReportResult report2 = reportResultService.create(context); report2.setType("healthcheck"); report2.setValue("{\"checks\":[]}"); + report2.setArgs("-c: 2, 3\n-r: reportout.csv\n"); reportResultService.update(context, report2); context.commit(); context.restoreAuthSystemState(); @@ -143,14 +144,20 @@ public void testShowDates() throws Exception { report2 = reportResultService.find(context, report2.getID()); TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "report-diff", "-d" }; + String[] args = new String[] { "report-diff", "-l" }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); List infoMessages = handler.getInfoMessages(); - assertThat(infoMessages, hasItem(containsString("Report Dates Summary:"))); + assertThat(infoMessages, hasItem(containsString("Available Reports Summary:"))); assertThat(infoMessages, hasItem(containsString("Report Type: healthcheck"))); + assertThat(infoMessages, hasItem(containsString("ID: " + report1.getID()))); + assertThat(infoMessages, hasItem(containsString("ID: " + report2.getID()))); assertThat(infoMessages, hasItem(containsString(formatDate(report1.getLastModified())))); assertThat(infoMessages, hasItem(containsString(formatDate(report2.getLastModified())))); + assertThat(infoMessages, hasItem(containsString("--check: 0 (General Information)"))); + assertThat(infoMessages, hasItem(containsString("--report: reportout.txt"))); + assertThat(infoMessages, hasItem(containsString("--for: 3"))); + assertThat(infoMessages, hasItem(containsString("--check: 2, 3"))); } @Test @@ -177,8 +184,8 @@ public void testCompareReports() throws Exception { report2 = reportResultService.find(context, report2.getID()); TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "report-diff", "-f", formatDate(report1.getLastModified()), - "-t", formatDate(report2.getLastModified()) }; + String[] args = new String[] { "report-diff", "-s", String.valueOf(report1.getID()), + "-t", String.valueOf(report2.getID()) }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); List infoMessages = handler.getInfoMessages(); @@ -193,9 +200,8 @@ public void testCompareSpecificCheck() throws Exception { ReportResult report1 = reportResultService.create(context); report1.setType("healthcheck"); - report1.setValue("{\"checks\":[{\"name\":\"Check1\",\"report\":{\"key\":\"value1\"}},{\"name\":\"Check2\"" + - ",\"report\":{\"key\":\"other\"}}]}"); - report1.setArgs("-c: 0"); + report1.setValue("{\"checks\":[{\"name\":\"General Information\",\"report\":{\"key\":\"value1\"}}," + + "{\"name\":\"Item summary\",\"report\":{\"key\":\"other\"}}]}"); reportResultService.update(context, report1); // Force commit and flush to ensure timestamp is set context.commit(); @@ -205,9 +211,8 @@ public void testCompareSpecificCheck() throws Exception { ReportResult report2 = reportResultService.create(context); report2.setType("healthcheck"); - report2.setValue("{\"checks\":[{\"name\":\"Check1\",\"report\":{\"key\":\"value2\"}},{\"name\":\"Check2\"" + - ",\"report\":{\"key\":\"other\"}}]}"); - report2.setArgs("-c: 0"); + report2.setValue("{\"checks\":[{\"name\":\"General Information\",\"report\":{\"key\":\"value2\"}}," + + "{\"name\":\"Item summary\",\"report\":{\"key\":\"other\"}}]}"); reportResultService.update(context, report2); context.commit(); context.restoreAuthSystemState(); @@ -215,8 +220,9 @@ public void testCompareSpecificCheck() throws Exception { report1 = reportResultService.find(context, report1.getID()); report2 = reportResultService.find(context, report2.getID()); TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "report-diff", "-f", formatDate(report1.getLastModified()), - "-t", formatDate(report2.getLastModified()), "-c", "0" }; + // -c 0 filters comparison to only General Information check + String[] args = new String[] { "report-diff", "-s", String.valueOf(report1.getID()), + "-t", String.valueOf(report2.getID()), "-c", "0" }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); List infoMessages = handler.getInfoMessages(); @@ -229,45 +235,46 @@ public void testCompareSpecificCheck() throws Exception { @Test public void testInvalidCheckIndex() throws Exception { TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "report-diff", "-f", "2023-01-01 00:00:00.000", - "-t", "2023-01-02 00:00:00.000", "-c", "999" }; + String[] args = new String[] { "report-diff", "-s", "1", "-t", "2", "-c", "999" }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); - List errorMessages = handler.getErrorMessages(); - assertThat(errorMessages, hasItem("Invalid value for check. Must be between 0 and " + - (HealthReport.getNumberOfChecks() - 1) + ". Using all checks.")); + // Invalid -c is now a warning + fallback (all checks compared), not a hard error. + List warningMessages = handler.getWarningMessages(); + assertThat(warningMessages, hasItem(containsString("Invalid value for -c: '999'"))); + assertThat(warningMessages, hasItem(containsString("All checks will be compared."))); } @Test - public void testInvalidDateFormat() throws Exception { + public void testInvalidReportIdFormat() throws Exception { TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "report-diff", "-f", "invalid-date", "-t", "2023-01-02 00:00:00.000" }; + String[] args = new String[] { "report-diff", "-s", "invalid-id", "-t", "2" }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); - List errorMessages = handler.getErrorMessages(); - assertThat(errorMessages, hasItem(containsString("Cannot create a Date from the input: invalid-date"))); + // Invalid -s value is now a warning and the missing ID falls back to latest report. + List warningMessages = handler.getWarningMessages(); + assertThat(warningMessages, hasItem(containsString("Invalid value for -s: 'invalid-id'"))); + assertThat(warningMessages, hasItem(containsString( + "The last report from the database will be used instead."))); } @Test - public void testNoReportsForDates() throws Exception { + public void testNoReportsForIds() throws Exception { TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "report-diff", "-f", "2022-01-01 00:00:00.000", - "-t", "2022-01-02 00:00:00.000" }; + String[] args = new String[] { "report-diff", "-s", "999999", "-t", "999998" }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); List infoMessages = handler.getInfoMessages(); - assertThat(infoMessages, hasItem(containsString("No reports found for specified dates."))); + assertThat(infoMessages, hasItem(containsString("No report found for report ID:"))); } @Test - public void testToBeforeFrom() throws Exception { + public void testNonPositiveReportId() throws Exception { TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "report-diff", "-f", "2023-01-02 00:00:00.000", - "-t", "2023-01-01 00:00:00.000" }; + String[] args = new String[] { "report-diff", "-s", "-1", "-t", "1" }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); List errorMessages = handler.getErrorMessages(); - assertThat(errorMessages, hasItem(containsString("The 'to' date cannot be before the 'from' date."))); + assertThat(errorMessages, hasItem(containsString("The 'source' report ID must be a positive integer."))); } @Test @@ -292,8 +299,8 @@ public void testReportWithMissingValue() throws Exception { report2 = reportResultService.find(context, report2.getID()); TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "report-diff", "-f", formatDate(report1.getLastModified()), - "-t", formatDate(report2.getLastModified()) }; + String[] args = new String[] { "report-diff", "-s", String.valueOf(report1.getID()), + "-t", String.valueOf(report2.getID()) }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); List infoMessages = handler.getInfoMessages(); @@ -323,12 +330,12 @@ public void testNoDifferences() throws Exception { report1 = reportResultService.find(context, report1.getID()); report2 = reportResultService.find(context, report2.getID()); TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "report-diff", "-f", formatDate(report1.getLastModified()), - "-t", formatDate(report2.getLastModified()) }; + String[] args = new String[] { "report-diff", "-s", String.valueOf(report1.getID()), + "-t", String.valueOf(report2.getID()) }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); List infoMessages = handler.getInfoMessages(); - assertThat(infoMessages, hasItem(containsString("No differences found."))); + assertThat(infoMessages, hasItem(containsString("No significant changes detected between reports."))); } @Test @@ -358,8 +365,8 @@ public void testNoEnteredDate() throws Exception { ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); List infoMessages = handler.getInfoMessages(); - assertThat(infoMessages, hasItem(containsString("No dates specified, " + - "using the last two dates from the database."))); + assertThat(infoMessages, hasItem(containsString("No report IDs specified, " + + "using the last two reports from the database."))); } @Test @@ -370,7 +377,6 @@ public void testReportDiff() throws Exception { report1.setType("healthcheck"); report1.setValue("{\"checks\":[{\"name\":\"Check1\",\"report\":{\"key\":\"value1\"}},{\"name\":\"Check2\"" + ",\"report\":{\"key\":\"other\"}}]}"); - report1.setArgs("-c: 0"); reportResultService.update(context, report1); // Force commit and flush to ensure timestamp is set @@ -383,7 +389,6 @@ public void testReportDiff() throws Exception { report2.setType("healthcheck"); report2.setValue("{\"checks\":[{\"name\":\"Check1\",\"report\":{\"key\":\"value2\"}},{\"name\":\"Check2\"" + ",\"report\":{\"key\":\"other\"}}]}"); - report2.setArgs("-c: 0"); reportResultService.update(context, report2); context.commit(); context.restoreAuthSystemState(); @@ -422,24 +427,24 @@ public void testShowDatesLimit() throws Exception { report2 = reportResultService.find(context, report2.getID()); TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "report-diff", "-d", "-l", "1" }; + String[] args = new String[] { "report-diff", "-l", "-m", "1" }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); List infoMessages = handler.getInfoMessages(); - assertThat(infoMessages, hasItem(containsString("Report Dates Summary:"))); + assertThat(infoMessages, hasItem(containsString("Available Reports Summary:"))); assertThat(infoMessages, hasItem(containsString("Report Type: healthcheck"))); - assertThat(infoMessages, not(hasItem(containsString(formatDate(report1.getLastModified()))))); - assertThat(infoMessages, hasItem(containsString(formatDate(report2.getLastModified())))); + assertThat(infoMessages, not(hasItem(containsString("ID: " + report1.getID())))); + assertThat(infoMessages, hasItem(containsString("ID: " + report2.getID()))); } @Test public void testProfessionalReportFormat() throws Exception { context.turnOffAuthorisationSystem(); - // Create first report with sample health data + // Create first report with sample health data using real check name ReportResult report1 = reportResultService.create(context); report1.setType("healthcheck"); - report1.setValue("{\"checks\":[{\"name\":\"HealthCheck\",\"report\":{" + + report1.setValue("{\"checks\":[{\"name\":\"General Information\",\"report\":{" + "\"publishedItems\":0," + "\"ePersonsCount\":1," + "\"communitiesCount\":0," + @@ -456,7 +461,7 @@ public void testProfessionalReportFormat() throws Exception { // Create second report with changes ReportResult report2 = reportResultService.create(context); report2.setType("healthcheck"); - report2.setValue("{\"checks\":[{\"name\":\"HealthCheck\",\"report\":{" + + report2.setValue("{\"checks\":[{\"name\":\"General Information\",\"report\":{" + "\"publishedItems\":2," + "\"ePersonsCount\":1721," + "\"communitiesCount\":9," + @@ -473,8 +478,8 @@ public void testProfessionalReportFormat() throws Exception { report2 = reportResultService.find(context, report2.getID()); TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "report-diff", "-f", formatDate(report1.getLastModified()), - "-t", formatDate(report2.getLastModified()) }; + String[] args = new String[] { "report-diff", "-s", String.valueOf(report1.getID()), + "-t", String.valueOf(report2.getID()) }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); List infoMessages = handler.getInfoMessages(); @@ -491,11 +496,11 @@ public void testProfessionalReportFormat() throws Exception { assertThat(infoMessages, hasItem(containsString("Key Changes"))); assertThat(infoMessages, hasItem(containsString("| Field"))); assertThat(infoMessages, hasItem(containsString("| Difference"))); - assertThat(infoMessages, hasItem(containsString("Assetstore Size (bytes)"))); - assertThat(infoMessages, hasItem(containsString("Log Directory Size (bytes)"))); + assertThat(infoMessages, hasItem(containsString("Assetstore Size"))); + assertThat(infoMessages, hasItem(containsString("Log Directory Size"))); // Test detailed change log section - assertThat(infoMessages, hasItem(containsString("Section 2: Detailed Change Log"))); + assertThat(infoMessages, hasItem(containsString("Section 3: Detailed Change Log"))); assertThat(infoMessages, hasItem(containsString("Changes Summary"))); assertThat(infoMessages, hasItem(containsString("Total operations:"))); assertThat(infoMessages, hasItem(containsString("Fields modified:"))); @@ -529,8 +534,8 @@ public void testReportFormatWithNoChanges() throws Exception { report2 = reportResultService.find(context, report2.getID()); TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "report-diff", "-f", formatDate(report1.getLastModified()), - "-t", formatDate(report2.getLastModified()) }; + String[] args = new String[] { "report-diff", "-s", String.valueOf(report1.getID()), + "-t", String.valueOf(report2.getID()) }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); List infoMessages = handler.getInfoMessages(); @@ -539,7 +544,6 @@ public void testReportFormatWithNoChanges() throws Exception { assertThat(infoMessages, hasItem(containsString("DSpace at My University: Repository Health Report Diff"))); assertThat(infoMessages, hasItem(containsString("Section 1: Executive Summary"))); assertThat(infoMessages, hasItem(containsString("No significant changes detected"))); - assertThat(infoMessages, hasItem(containsString("No differences found."))); } @Test @@ -566,8 +570,8 @@ public void testCalculateTimePeriod() throws Exception { report2 = reportResultService.find(context, report2.getID()); TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "report-diff", "-f", formatDate(report1.getLastModified()), - "-t", formatDate(report2.getLastModified()) }; + String[] args = new String[] { "report-diff", "-s", String.valueOf(report1.getID()), + "-t", String.valueOf(report2.getID()) }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); List infoMessages = handler.getInfoMessages(); @@ -584,10 +588,10 @@ public void testCalculateTimePeriod() throws Exception { public void testEnhancedKeyChangesTable() throws Exception { context.turnOffAuthorisationSystem(); - // Create first report with sample health data + // Create first report with sample health data using real check names ReportResult report1 = reportResultService.create(context); report1.setType("healthcheck"); - report1.setValue("{\"checks\":[{\"name\":\"Info summary\",\"report\":{}}," + + report1.setValue("{\"checks\":[{\"name\":\"General Information\",\"report\":{}}," + "{\"name\":\"Item summary\",\"report\":{" + "\"publishedItems\":10," + "\"ePersonsCount\":5," + @@ -604,7 +608,7 @@ public void testEnhancedKeyChangesTable() throws Exception { // Create second report with changes ReportResult report2 = reportResultService.create(context); report2.setType("healthcheck"); - report2.setValue("{\"checks\":[{\"name\":\"Info summary\",\"report\":{}}," + + report2.setValue("{\"checks\":[{\"name\":\"General Information\",\"report\":{}}," + "{\"name\":\"Item summary\",\"report\":{" + "\"publishedItems\":25," + "\"ePersonsCount\":8," + @@ -621,8 +625,8 @@ public void testEnhancedKeyChangesTable() throws Exception { report2 = reportResultService.find(context, report2.getID()); TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "report-diff", "-f", formatDate(report1.getLastModified()), - "-t", formatDate(report2.getLastModified()) }; + String[] args = new String[] { "report-diff", "-s", String.valueOf(report1.getID()), + "-t", String.valueOf(report2.getID()) }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); List infoMessages = handler.getInfoMessages(); @@ -662,12 +666,16 @@ public void testSizeDifferenceFormatting() throws Exception { // Create reports with size differences from "0 bytes" to "9 KB" String fromReportJson = "{ \"checks\": [" + - " { \"report\": { \"totalSize\": \"0 bytes\" } }" + + " { \"name\": \"Item summary\", \"report\": { " + + " \"collectionsSizesInfo\": { \"totalSize\": \"0 bytes\" }" + + " } }" + "]}"; String toReportJson = "{ \"checks\": [" + - " { \"report\": { \"totalSize\": \"9 KB\" } }" + + " { \"name\": \"Item summary\", \"report\": { " + + " \"collectionsSizesInfo\": { \"totalSize\": \"9 KB\" }" + + " } }" + "]}"; ReportResult fromReport = reportResultService.create(context); @@ -694,19 +702,315 @@ public void testSizeDifferenceFormatting() throws Exception { String[] args = new String[] { "report-diff", - "-f", formatDate(fromReport.getLastModified()), - "-t", formatDate(toReport.getLastModified()) + "-s", String.valueOf(fromReport.getID()), + "-t", String.valueOf(toReport.getID()) }; ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), testHandler, kernelImpl); List infoMessages = testHandler.getInfoMessages(); - // Verify that size differences show actual byte differences instead of "Changed" + // Verify that size differences show actual size delta instead of "Changed" boolean hasSizeDifference = infoMessages.stream() - .anyMatch(msg -> msg.contains("totalSize") && msg.contains("9 KB")); + .anyMatch(msg -> msg.contains("Total Content Size") && msg.contains("+9 KB")); - assertThat("Size differences should show actual size change (9 KB).'", + assertThat("Size differences should show actual size change (+9 KB).", hasSizeDifference, org.hamcrest.Matchers.is(true)); } + + @Test + public void testSkippedChecksSection() throws Exception { + context.turnOffAuthorisationSystem(); + + // Create report1 with checks A and B + ReportResult report1 = reportResultService.create(context); + report1.setType("healthcheck"); + report1.setValue("{\"checks\":[" + + "{\"name\":\"General Information\",\"report\":{\"key\":\"val1\"}}," + + "{\"name\":\"Only In From\",\"report\":{\"key\":\"fromOnly\"}}" + + "]}"); + reportResultService.update(context, report1); + context.commit(); + + Thread.sleep(1000); + + // Create report2 with checks A and C (B missing, C new) + ReportResult report2 = reportResultService.create(context); + report2.setType("healthcheck"); + report2.setValue("{\"checks\":[" + + "{\"name\":\"General Information\",\"report\":{\"key\":\"val2\"}}," + + "{\"name\":\"Only In To\",\"report\":{\"key\":\"toOnly\"}}" + + "]}"); + reportResultService.update(context, report2); + context.commit(); + context.restoreAuthSystemState(); + + report1 = reportResultService.find(context, report1.getID()); + report2 = reportResultService.find(context, report2.getID()); + + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "report-diff", "-s", String.valueOf(report1.getID()), + "-t", String.valueOf(report2.getID()) }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + List infoMessages = handler.getInfoMessages(); + + // Should show the skipped checks section + assertThat(infoMessages, hasItem(containsString("Skipped Checks"))); + assertThat(infoMessages, hasItem(containsString("not present in both reports"))); + assertThat(infoMessages, hasItem(containsString("Only In From"))); + assertThat(infoMessages, hasItem(containsString("Only In To"))); + + // The common check "General Information" should be compared normally + assertThat("Should contain diff for common check", + hasDiffOperation(infoMessages, "REPLACE", CHECK_KEY_PATH), + org.hamcrest.Matchers.is(true)); + } + + @Test + public void testCompareReportsWithMissingMappedFieldDoesNotFail() throws Exception { + context.turnOffAuthorisationSystem(); + + ReportResult report1 = reportResultService.create(context); + report1.setType("healthcheck"); + report1.setValue("{\"checks\":[{\"name\":\"Item summary\",\"report\":{}}]}"); + reportResultService.update(context, report1); + context.commit(); + + Thread.sleep(1000); + + ReportResult report2 = reportResultService.create(context); + report2.setType("healthcheck"); + report2.setValue("{\"checks\":[{\"name\":\"Item summary\",\"report\":{" + + "\"publishedItems\":2" + + "}}]}"); + reportResultService.update(context, report2); + context.commit(); + context.restoreAuthSystemState(); + + report1 = reportResultService.find(context, report1.getID()); + report2 = reportResultService.find(context, report2.getID()); + + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "report-diff", "-s", String.valueOf(report1.getID()), + "-t", String.valueOf(report2.getID()) }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + assertThat("Script should not fail with missing mapped field", handler.getErrorMessages(), empty()); + assertThat(handler.getInfoMessages(), hasItem(containsString("Repository Health Report Diff"))); + assertThat(handler.getInfoMessages(), hasItem(containsString("Published Items"))); + } + + /** + * When only -s is provided (no -t), the script should warn the user and set -t to + * the latest report in the database. + */ + @Test + public void testSourceWithoutTargetWarnsAndFallsBack() throws Exception { + context.turnOffAuthorisationSystem(); + + ReportResult report1 = reportResultService.create(context); + report1.setType("healthcheck"); + report1.setValue("{\"checks\":[{\"name\":\"Check1\",\"report\":{\"key\":\"value1\"}}]}"); + reportResultService.update(context, report1); + context.commit(); + Thread.sleep(1000); + + ReportResult report2 = reportResultService.create(context); + report2.setType("healthcheck"); + report2.setValue("{\"checks\":[{\"name\":\"Check1\",\"report\":{\"key\":\"value2\"}}]}"); + reportResultService.update(context, report2); + context.commit(); + context.restoreAuthSystemState(); + + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "report-diff", "-s", String.valueOf(report1.getID()) }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + // When only -s is supplied, the missing -t is now auto-filled with the latest report. + assertThat(handler.getInfoMessages(), hasItem(containsString( + "Only '-s' was specified; '-t' will be set to the latest report from the database."))); + assertThat(handler.getErrorMessages(), empty()); + } + + /** + * When only -t is provided (no -s), the script should warn the user and set -s to + * the latest report in the database. + */ + @Test + public void testTargetWithoutSourceWarnsAndFallsBack() throws Exception { + context.turnOffAuthorisationSystem(); + + ReportResult report1 = reportResultService.create(context); + report1.setType("healthcheck"); + report1.setValue("{\"checks\":[{\"name\":\"Check1\",\"report\":{\"key\":\"value1\"}}]}"); + reportResultService.update(context, report1); + context.commit(); + Thread.sleep(1000); + + ReportResult report2 = reportResultService.create(context); + report2.setType("healthcheck"); + report2.setValue("{\"checks\":[{\"name\":\"Check1\",\"report\":{\"key\":\"value2\"}}]}"); + reportResultService.update(context, report2); + context.commit(); + context.restoreAuthSystemState(); + + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "report-diff", "-t", String.valueOf(report2.getID()) }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + // When only -t is supplied, the missing -s is now auto-filled with the next latest + // report (instead of falling back to the last two reports). The script logs a dedicated + // info message announcing that and the comparison still runs without errors. + assertThat(handler.getInfoMessages(), hasItem(containsString( + "Only '-t' was specified; '-s' will be set to the latest report from the database."))); + assertThat(handler.getErrorMessages(), empty()); + } + + /** + * When -s has a non-numeric value, the script should warn and default missing IDs + * to latest report values. + */ + @Test + public void testInvalidSourceValueWarnsAndFallsBack() throws Exception { + context.turnOffAuthorisationSystem(); + + ReportResult report1 = reportResultService.create(context); + report1.setType("healthcheck"); + report1.setValue("{\"checks\":[{\"name\":\"Check1\",\"report\":{\"key\":\"value1\"}}]}"); + reportResultService.update(context, report1); + context.commit(); + Thread.sleep(1000); + + ReportResult report2 = reportResultService.create(context); + report2.setType("healthcheck"); + report2.setValue("{\"checks\":[{\"name\":\"Check1\",\"report\":{\"key\":\"value2\"}}]}"); + reportResultService.update(context, report2); + context.commit(); + context.restoreAuthSystemState(); + + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "report-diff", "-s", "abc", "-t", String.valueOf(report2.getID()) }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + assertThat(handler.getWarningMessages(), hasItem(containsString("Invalid value for -s: 'abc'"))); + assertThat(handler.getWarningMessages(), hasItem(containsString( + "The last report from the database will be used instead."))); + // Source becomes null after the invalid -s parse, so the missing-source branch in + // defaultReportIds now logs an info message instead of the legacy XOR warning. + assertThat(handler.getInfoMessages(), hasItem(containsString( + "Only '-t' was specified; '-s' will be set to the latest report from the database."))); + } + + /** + * When -s points to a non-existing report, the script aborts with "No report found" + * and does not log the defaulting message for the missing -t. + */ + @Test + public void testNonExistingSourceIdAbortsWithoutDefaulting() throws Exception { + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "report-diff", "-s", "999999" }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + assertThat(handler.getInfoMessages(), hasItem(containsString("No report found for report ID: 999999"))); + assertThat(handler.getInfoMessages(), not(hasItem(containsString( + "Only '-s' was specified; '-t' will be set to the latest report from the database.")))); + } + + /** + * When -c is an out-of-range index, the script should warn and compare all checks + * instead of filtering to one. + */ + @Test + public void testInvalidCheckIndexWarnsAndComparesAll() throws Exception { + context.turnOffAuthorisationSystem(); + + ReportResult report1 = reportResultService.create(context); + report1.setType("healthcheck"); + report1.setValue("{\"checks\":[{\"name\":\"General Information\",\"report\":{\"key\":\"v1\"}}]}"); + reportResultService.update(context, report1); + context.commit(); + Thread.sleep(1000); + + ReportResult report2 = reportResultService.create(context); + report2.setType("healthcheck"); + report2.setValue("{\"checks\":[{\"name\":\"General Information\",\"report\":{\"key\":\"v2\"}}]}"); + reportResultService.update(context, report2); + context.commit(); + context.restoreAuthSystemState(); + + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "report-diff", + "-s", String.valueOf(report1.getID()), + "-t", String.valueOf(report2.getID()), + "-c", "999" }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + assertThat(handler.getWarningMessages(), hasItem(containsString("Invalid value for -c: '999'"))); + assertThat(handler.getWarningMessages(), hasItem(containsString("All checks will be compared."))); + // Comparison still runs and produces the diff for the common check. + assertThat(handler.getInfoMessages(), hasItem(containsString("Repository Health Report Diff"))); + assertThat(handler.getErrorMessages(), empty()); + } + + /** + * When -c is a non-numeric value, the script should warn and compare all checks. + */ + @Test + public void testNonNumericCheckIndexWarnsAndComparesAll() throws Exception { + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "report-diff", "-s", "1", "-t", "2", "-c", "abc" }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + assertThat(handler.getWarningMessages(), hasItem(containsString("Invalid value for -c: 'abc'"))); + assertThat(handler.getWarningMessages(), hasItem(containsString("All checks will be compared."))); + } + + /** + * When -m has a non-numeric value alongside -l, the script should warn and show + * all available entries (no limit). + */ + @Test + public void testInvalidMaxValueWarnsAndShowsAll() throws Exception { + context.turnOffAuthorisationSystem(); + + ReportResult report1 = reportResultService.create(context); + report1.setType("healthcheck"); + report1.setValue("{\"checks\":[]}"); + reportResultService.update(context, report1); + context.commit(); + Thread.sleep(1000); + + ReportResult report2 = reportResultService.create(context); + report2.setType("healthcheck"); + report2.setValue("{\"checks\":[]}"); + reportResultService.update(context, report2); + context.commit(); + context.restoreAuthSystemState(); + + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "report-diff", "-l", "-m", "abc" }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + assertThat(handler.getWarningMessages(), hasItem(containsString("Invalid value for -m: 'abc'"))); + assertThat(handler.getWarningMessages(), hasItem(containsString("All entries will be shown."))); + // Both reports must be present in the listing since the limit was discarded. + assertThat(handler.getInfoMessages(), hasItem(containsString("ID: " + report1.getID()))); + assertThat(handler.getInfoMessages(), hasItem(containsString("ID: " + report2.getID()))); + assertThat(handler.getErrorMessages(), empty()); + } + + /** + * When -m has a non-positive value, same fallback as non-numeric: warn and show all. + */ + @Test + public void testNonPositiveMaxValueWarnsAndShowsAll() throws Exception { + TestDSpaceRunnableHandler handler = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "report-diff", "-l", "-m", "0" }; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), handler, kernelImpl); + + assertThat(handler.getWarningMessages(), hasItem(containsString("Invalid value for -m: '0'"))); + assertThat(handler.getWarningMessages(), hasItem(containsString("All entries will be shown."))); + assertThat(handler.getErrorMessages(), empty()); + } } diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ScriptRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ScriptRestRepository.java index 1eea06a4ee8d..c0e86e0353c3 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ScriptRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ScriptRestRepository.java @@ -32,6 +32,7 @@ import org.dspace.core.Context; import org.dspace.scripts.DSpaceCommandLineParameter; import org.dspace.scripts.DSpaceRunnable; +import org.dspace.scripts.DSpaceRunnable.StepResult; import org.dspace.scripts.configuration.ScriptConfiguration; import org.dspace.scripts.service.ScriptService; import org.springframework.beans.factory.annotation.Autowired; @@ -151,7 +152,14 @@ private void runDSpaceScript(List files, Context context, ScriptC throws IOException, SQLException, AuthorizeException, InstantiationException, IllegalAccessException { DSpaceRunnable dSpaceRunnable = scriptService.createDSpaceRunnableForScriptConfiguration(scriptToExecute); try { - dSpaceRunnable.initialize(args.toArray(new String[0]), restDSpaceRunnableHandler, context.getCurrentUser()); + StepResult initResult = dSpaceRunnable.initialize( + args.toArray(new String[0]), restDSpaceRunnableHandler, context.getCurrentUser()); + // -h/--help returns Exit: skip run() and just mark the process started + completed. + if (initResult == StepResult.Exit) { + restDSpaceRunnableHandler.start(); + restDSpaceRunnableHandler.handleCompletion(); + return; + } if (files != null && !files.isEmpty()) { checkFileNames(dSpaceRunnable, files); processFiles(context, restDSpaceRunnableHandler, files); diff --git a/dspace/config/launcher.xml b/dspace/config/launcher.xml index 3853b4e2fabd..dba2f2f73e1b 100644 --- a/dspace/config/launcher.xml +++ b/dspace/config/launcher.xml @@ -7,13 +7,7 @@ org.dspace.storage.bitstore.BitStoreMigrate - - healthcheck - Create health check report - - org.dspace.health.Report - - + checker Run the checksum checker From 8f4b80d514a59a59b1c5032800da873e7ce4606c Mon Sep 17 00:00:00 2001 From: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> Date: Wed, 17 Jun 2026 12:26:12 +0200 Subject: [PATCH 29/41] Fix flaky tests in IT pipeline (#1321) * Fixed integration tests because they use to fail sometimes * test: stabilize flaky CI tests (Hibernate cleanup retry, Shibboleth auth sequence reset, ORCID assertion hardening) * test: fix flaky ITs at the source (live ORCID, Shibboleth config-reload) + Hibernate CME diagnostics ORCID CachingOrcidRestConnectorTest no longer hits the live ORCID sandbox: search/getLabel/search_fail mock the HTTP layer (httpGet made protected) with a canned expanded-search response, so they are deterministic instead of asserting against fluctuating sandbox data. Shibboleth WWW-Authenticate flakiness: add a test-only config-definition.xml with config-reload=false. Runtime setProperty(...AuthenticationMethod...) overrides were silently discarded whenever the auto-reload listener rebuilt the combined config (restoring clarin-dspace.cfg's [Password, ClarinShib] default), intermittently leaking 'password realm' into the header. Verified: with auto-reload off the override survives; the explicit reloadConfig() reset in @After still works. Hibernate ConcurrentModificationException in @After cleanup: the per-session JDBC ResourceRegistry is not thread-safe, so the CME means two threads touch one Session. Capture a full thread dump on CME (target/cme-dumps/) to identify the colliding thread in CI; keep a resilient retry so an already-passed test isn't failed by this teardown race. (Context.finalize() ruled out: sessions are thread-local.) Co-Authored-By: Claude Opus 4.8 * test: revert IT-env config-reload=false override Disabling config auto-reload globally in the test environment broke AuthorizeConfigIT.testReloadConfiguration, which deliberately verifies that AuthorizeConfiguration picks up live changes written to local.cfg via the auto-reload mechanism. Auto-reload is a tested feature here, so it must not be disabled to work around the Shibboleth WWW-Authenticate flakiness. The Shibboleth flakiness (runtime setProperty override discarded when the combined config is rebuilt) needs a reload-safe fix in the auth test instead; tracked separately. Co-Authored-By: Claude Opus 4.8 * test: make Shibboleth auth-sequence override reload-safe (fix WWW-Authenticate flakiness) The flaky 'password realm' leak in AuthenticationRestControllerIT had this root cause: configurationService.setProperty(plugin.sequence...AuthenticationMethod, ...) only updates the in-memory view of the combined configuration. That view is discarded whenever it is rebuilt, and the auto-reload listener rebuilds it as soon as any reloadable cfg file's mtime changes mid-run (e.g. another test writing local.cfg). When that rebuild lands between the override and the request, clarin-dspace.cfg's default [PasswordAuthentication, ClarinShibAuthentication] returns and 'password realm' leaks into the header. The previous clear-then-set helper did not help (it is equivalent to a plain setProperty). Fix: set the sequence via a JVM system property (highest-precedence override layer, re-read on every rebuild) + reloadConfig(), and clear it in @After. This survives auto-reload without disabling it (so AuthorizeConfigIT, which verifies auto-reload, still passes). Verified in the real /api/authn/status endpoint: an explicit reloadConfig() after a setProperty override reproduces the leak, while the system-property approach keeps the header Shibboleth-only across rebuilds. Full AuthenticationRestControllerIT (43 tests) passes, and running it alongside ClarinAuthenticationRestControllerIT / AnonymousAdditionalAuthorizationFilterIT confirms the property does not leak across classes. Co-Authored-By: Claude Opus 4.8 * test: add Hibernate concurrency monitor + CI upload to pinpoint @After CME The intermittent ConcurrentModificationException in @After cleanup is a genuine cross-thread data race on Hibernate's per-session, non-thread-safe JDBC ResourceRegistry (xref): a second thread mutates the test thread's session while it commits/rolls back. Verified against hibernate-core-5.6.15 sources that the releaseResources forEach lambda never touches xref, so single-thread re-entrancy is impossible (this disproves the earlier HHH-15116 single-thread theory). The window is microseconds, so it does not reproduce locally even with deliberate cross-thread session sharing; it only surfaces under CI load. A live thread dump of a running IT JVM shows NO legitimate background thread ever touches Hibernate (all are Solr/HTTP/Jetty/JVM). So the culprit is a transient thread, and any non-test thread caught inside Hibernate JDBC/session code is by definition the offender. - HibernateConcurrencyMonitor: JVM-wide background sampler that records (de-duped) any non-test thread found inside org.hibernate.{resource.jdbc,engine.jdbc, internal.SessionImpl}; flushed to target/cme-dumps/ on CME and at JVM shutdown. Pure observer, never changes test behaviour. - AbstractIntegrationTestWithDatabase: start the monitor and mark the JUnit thread in setUp; flush it alongside the existing thread dump on a captured CME. - build.yml: always-upload **/target/cme-dumps/** (not gated on failure) so a successful cleanup retry no longer hides the diagnostic. Co-Authored-By: Claude Opus 4.8 * fix: don't close iterate() Hibernate stream from a finalize() (root cause of flaky CME) Root cause of the intermittent ConcurrentModificationException in @After integration-test cleanup, identified via the HibernateConcurrencyMonitor CI dumps: the GC Finalizer thread, running org.dspace.core.AbstractHibernateDAO$1.finalize(), closed the Hibernate Stream returned by AbstractHibernateDAO.iterate(). Closing a stream closes its ScrollableResults, which mutates the owning Session's per-session, non-thread-safe JDBC ResourceRegistry (xref) - but on the Finalizer thread, concurrently with the thread that owns the session. When that collided with the owning thread's commit/rollback (releaseResources -> xref.forEach), it threw ConcurrentModificationException. The CI dumps showed this exact finalizer stack as the only non-test thread inside Hibernate in dspace-api, and present in dspace-server-webapp too. This was confirmed genuine cross-thread access (not the previously assumed single-thread/HHH bug): verified against hibernate-core-5.6.15 sources that the releaseResources forEach lambda never touches xref, so single-thread re-entrancy is impossible. Fix: close the backing stream on the owning thread when iteration is exhausted, and remove the finalize() override. An iterator abandoned before exhaustion is released safely when its Context/Session is closed (releaseResources then runs on the owning thread). Adds AbstractHibernateDAOIteratorIT to guard against reintroducing a stream-closing finalizer. Co-Authored-By: Claude Opus 4.8 * fix: remove broken Context.finalize() that leaked finalizer-thread sessions Context.finalize() ran on the GC Finalizer thread and called dbConnection.isTransActionAlive()/abort(), which resolve sessionFactory.getCurrentSession() to a brand-new session bound to the Finalizer thread - never the (now-unreachable) thread that opened the Context. So it could not roll back the Context's transaction anyway; it only opened and leaked a throwaway Hibernate session on the Finalizer thread, and threw IllegalStateException once the SessionFactory was closed (seen in the CI thread dumps used to diagnose the flaky integration-test ConcurrentModificationException). Abandoned Contexts are cleaned up safely when their owning thread's session ends; callers already close Contexts via complete()/abort()/try-with-resources (Context is AutoCloseable). Removes the now-redundant ContextTest.testFinalize (close()/abort() are covered by testClose/testAbort/testAbort2). Co-Authored-By: Claude Opus 4.8 * test: remove flaky-CME diagnostic scaffolding and teardown retry (root cause fixed) The intermittent @After ConcurrentModificationException is now fixed at its source (AbstractHibernateDAO.iterate no longer closes its Hibernate stream from a finalizer; broken Context.finalize() removed). The temporary diagnostics that pinpointed it are no longer needed: - Restore AbstractIntegrationTestWithDatabase.destroy() to its plain form (drop the 3x cleanup retry and the per-CME thread dump) and remove the HibernateConcurrencyMonitor wiring. - Delete HibernateConcurrencyMonitor. - Revert the build.yml always-upload of target/cme-dumps. CI keeps -Dfailsafe.rerunFailingTestsCount=2 as the generic flaky-test safety net. Co-Authored-By: Claude Opus 4.8 * revert: keep Context.finalize() (out of scope, not the CME cause) Reverts the Context.finalize() removal (and the ContextTest.testFinalize deletion). The flaky @After ConcurrentModificationException is fully fixed by the AbstractHibernateDAO.iterate() change alone; Context.finalize() runs on a single GC Finalizer thread against its own finalizer-thread session and provably cannot cause that cross-thread xref race. Removing a finalizer from this core, widely-used class is a riskier change that does not belong in a flaky-test fix, so leave Context untouched. The (pre-existing, harmless) finalizer-thread session it opens can be addressed separately if desired. Co-Authored-By: Claude Opus 4.8 * test: address review comments on flaky-test fix - AbstractHibernateDAOIteratorIT: add Javadoc to the test method and walk the iterator's full class hierarchy (up to Object) when asserting no finalize() override, so a finalizer reintroduced on a superclass/helper is also caught (per CodeRabbit review). - AuthenticationRestControllerIT: wrap an over-length (122 char) Javadoc line. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- .../org/dspace/core/AbstractHibernateDAO.java | 20 ++++- .../external/CachingOrcidRestConnector.java | 4 +- .../core/AbstractHibernateDAOIteratorIT.java | 76 +++++++++++++++++++ .../CachingOrcidRestConnectorTest.java | 46 +++++++++-- .../dspace/external/orcid-expanded-search.xml | 22 ++++++ .../rest/AuthenticationRestControllerIT.java | 76 ++++++++++++++----- 6 files changed, 214 insertions(+), 30 deletions(-) create mode 100644 dspace-api/src/test/java/org/dspace/core/AbstractHibernateDAOIteratorIT.java create mode 100644 dspace-api/src/test/resources/org/dspace/external/orcid-expanded-search.xml diff --git a/dspace-api/src/main/java/org/dspace/core/AbstractHibernateDAO.java b/dspace-api/src/main/java/org/dspace/core/AbstractHibernateDAO.java index 32ad747d765e..498c52c27f46 100644 --- a/dspace-api/src/main/java/org/dspace/core/AbstractHibernateDAO.java +++ b/dspace-api/src/main/java/org/dspace/core/AbstractHibernateDAO.java @@ -306,11 +306,23 @@ public Iterator iterate(Query query) { return new AbstractIterator () { @Override protected T computeNext() { - return iter.hasNext() ? iter.next() : endOfData(); - } - @Override - public void finalize() { + if (iter.hasNext()) { + return iter.next(); + } + // Close the backing ScrollableResults / JDBC cursor as soon as the iteration is exhausted, on + // the thread that owns the Hibernate Session (the caller's thread). + // + // This MUST NOT be done from a finalize() override (as it previously was): finalize() runs on + // the GC Finalizer thread, so closing the stream there mutates the Session's per-session, + // non-thread-safe JDBC ResourceRegistry (xref) concurrently with the owning thread. That is a + // genuine data race which intermittently throws ConcurrentModificationException from + // ResourceRegistryStandardImpl.releaseResources during an unrelated commit/rollback (observed + // as flaky integration-test failures in AbstractIntegrationTestWithDatabase teardown). + // + // An iterator abandoned before exhaustion no longer leaks: its open statement is released + // safely when the owning Context/Session is closed (releaseResources runs on the owning thread). stream.close(); + return endOfData(); } }; } diff --git a/dspace-api/src/main/java/org/dspace/external/CachingOrcidRestConnector.java b/dspace-api/src/main/java/org/dspace/external/CachingOrcidRestConnector.java index e34767a25063..4eee572d5a6f 100644 --- a/dspace-api/src/main/java/org/dspace/external/CachingOrcidRestConnector.java +++ b/dspace-api/src/main/java/org/dspace/external/CachingOrcidRestConnector.java @@ -174,7 +174,9 @@ protected String getAccessToken(String clientSecret, String clientId, String OAU } } - private InputStream httpGet(String path, String accessToken) throws IOException { + // Package/sub-class visible so tests can stub the HTTP layer (see CachingOrcidRestConnectorTest) + // and avoid hitting the live ORCID sandbox. + protected InputStream httpGet(String path, String accessToken) throws IOException { String trimmedPath = path.replaceFirst("^/+", "").replaceFirst("/+$", ""); String fullPath = apiURL + '/' + trimmedPath; diff --git a/dspace-api/src/test/java/org/dspace/core/AbstractHibernateDAOIteratorIT.java b/dspace-api/src/test/java/org/dspace/core/AbstractHibernateDAOIteratorIT.java new file mode 100644 index 000000000000..fcc00434d620 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/core/AbstractHibernateDAOIteratorIT.java @@ -0,0 +1,76 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.core; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.util.Iterator; + +import org.dspace.AbstractIntegrationTestWithDatabase; +import org.dspace.content.MetadataValue; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.MetadataValueService; +import org.junit.Test; + +/** + * Regression test for the intermittent {@code ConcurrentModificationException} thrown during + * {@code @After} integration-test cleanup and traced to + * {@link AbstractHibernateDAO#iterate(javax.persistence.Query)}. + * + *

That method used to close its Hibernate {@code Stream} from a {@code finalize()} override. {@code finalize()} + * runs on the GC Finalizer thread, so closing the stream there mutated the owning {@code Session}'s per-session, + * non-thread-safe JDBC {@code ResourceRegistry} (xref) concurrently with the thread that owns the session. That + * is a genuine data race which intermittently threw {@code ConcurrentModificationException} from + * {@code ResourceRegistryStandardImpl.releaseResources} during an unrelated commit/rollback. The fix closes the + * stream on the owning thread once the iteration is exhausted. This test guards against reintroducing any + * stream-closing finalizer on the returned iterator.

+ */ +public class AbstractHibernateDAOIteratorIT extends AbstractIntegrationTestWithDatabase { + + private final MetadataValueService metadataValueService = + ContentServiceFactory.getInstance().getMetadataValueService(); + + /** + * Verifies that the iterator returned by {@link AbstractHibernateDAO#iterate(javax.persistence.Query)} + * (exercised here through {@code MetadataValueService.findByValueLike}) does not close its backing Hibernate + * stream from a {@code finalize()} override anywhere in its class hierarchy, and that it still iterates to + * exhaustion (closing its cursor on the owning thread) without error. No matching rows are required - the + * wrapper iterator is created regardless of the result count. + * + * @throws Exception passed through. + */ + @Test + public void iterateIteratorMustNotCloseStreamFromFinalizer() throws Exception { + Iterator iterator = + metadataValueService.findByValueLike(context, "no-such-metadata-value-" + System.nanoTime()); + assertNotNull(iterator); + + // The returned iterator - and every class in its hierarchy up to Object - MUST NOT declare a finalize() + // override: closing the backing Hibernate Stream from the GC Finalizer thread is exactly the cross-thread + // access to the non-thread-safe per-session JDBC ResourceRegistry that caused the flaky + // ConcurrentModificationException. Walking the hierarchy also catches a finalizer reintroduced on a + // superclass/helper rather than on the anonymous leaf class. + for (Class type = iterator.getClass(); type != null && type != Object.class; type = type.getSuperclass()) { + try { + type.getDeclaredMethod("finalize"); + fail("AbstractHibernateDAO.iterate() iterator must not declare a finalize() override (found on " + + type.getName() + ") - closing the Hibernate Stream on the GC Finalizer thread races the " + + "owning thread's non-thread-safe JDBC ResourceRegistry and intermittently throws " + + "ConcurrentModificationException."); + } catch (NoSuchMethodException expected) { + // good: no stream-closing finalizer on this class + } + } + + // It must still iterate to exhaustion and close its cursor on THIS (the owning) thread without error. + while (iterator.hasNext()) { + assertNotNull(iterator.next()); + } + } +} diff --git a/dspace-api/src/test/java/org/dspace/external/CachingOrcidRestConnectorTest.java b/dspace-api/src/test/java/org/dspace/external/CachingOrcidRestConnectorTest.java index bdb051601cb8..7e8cbc6c94fc 100644 --- a/dspace-api/src/test/java/org/dspace/external/CachingOrcidRestConnectorTest.java +++ b/dspace-api/src/test/java/org/dspace/external/CachingOrcidRestConnectorTest.java @@ -13,9 +13,13 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import java.io.IOException; +import java.io.InputStream; + import org.dspace.AbstractDSpaceTest; import org.dspace.external.provider.orcid.xml.ExpandedSearchConverter; import org.dspace.utils.DSpace; @@ -33,8 +37,22 @@ public class CachingOrcidRestConnectorTest extends AbstractDSpaceTest { private static final String orcid = "0000-0002-9150-2529"; private static final String expectedLabel = "Connor, John"; + // Canned ORCID "expanded-search" response (num-found=1725, first result -> "Connor, John"). + // Used to mock the HTTP layer so the tests don't depend on the live ORCID sandbox. + private static final String EXPANDED_SEARCH_XML = "org/dspace/external/orcid-expanded-search.xml"; + private CachingOrcidRestConnector sut; + /** + * Load a canned API response from the test classpath as a fresh InputStream. + * (A new stream is returned on every call because the connector consumes/closes it.) + */ + private InputStream cannedResponse(String resource) { + InputStream is = getClass().getClassLoader().getResourceAsStream(resource); + assertNotNull("Missing test resource: " + resource, is); + return is; + } + @Before public void setup() { sut = new CachingOrcidRestConnector(); @@ -59,40 +77,54 @@ public void getAccessToken() { } @Test - public void getLabel() { + public void getLabel() throws Exception { sut = Mockito.spy(sut); sut.setApiURL("https://pub.sandbox.orcid.org/v3.0"); //Mock the CachingOrcidRestConnector so that getAccessToken returns sandboxToken doReturn(sandboxToken).when(sut).getAccessToken(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + //Mock the HTTP layer with a canned response so we don't depend on the live ORCID sandbox. + doReturn(cannedResponse(EXPANDED_SEARCH_XML)).when(sut).httpGet(Mockito.anyString(), Mockito.anyString()); String label = sut.getLabel(orcid); assertEquals(expectedLabel, label); } @Test - public void search() { + public void search() throws Exception { sut = Mockito.spy(sut); sut.setApiURL("https://pub.sandbox.orcid.org/v3.0"); //Mock the CachingOrcidRestConnector so that getAccessToken returns sandboxToken doReturn(sandboxToken).when(sut).getAccessToken(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + //Mock the HTTP layer with a canned ORCID expanded-search response. Previously this test hit the live + //ORCID sandbox and asserted numFound() > 1000, which flaked whenever the sandbox dataset was reset/shrunk. + //Mocking the transport keeps the real parsing + edismax wildcard query-building path under test, but makes + //the result deterministic. + doReturn(cannedResponse(EXPANDED_SEARCH_XML)).when(sut).httpGet(Mockito.anyString(), Mockito.anyString()); ExpandedSearchConverter.Results search = sut.search("joh", 0, 1); - //Should match all Johns also, because edismax with wildcard - assertTrue(search.numFound() > 1000); + assertTrue("Expected a successful ORCID response, got: " + search, search.isOk()); + //'joh' is alphabetic, so the connector turns it into an edismax wildcard query ("joh || joh*") that matches + //many authors; the canned response carries num-found=1725. + assertEquals("Unexpected num-found for the canned ORCID response", 1725L, (long) search.numFound()); + assertEquals("Connor, John", search.results().get(0).label()); } @Test - public void search_fail() { + public void search_fail() throws Exception { sut = Mockito.spy(sut); sut.setApiURL("https://pub.sandbox.orcid.org/v3.0"); - //Mock the CachingOrcidRestConnector so that getAccessToken returns and invalid token + //Mock the CachingOrcidRestConnector so that getAccessToken returns an invalid token doReturn("FAKE").when(sut).getAccessToken(Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + //Simulate the ORCID API rejecting the (fake) token: every httpGet fails. Done via the mocked HTTP layer + //so the test is deterministic and doesn't rely on the live sandbox returning a 401. + doThrow(new IOException("simulated ORCID auth failure")).when(sut) + .httpGet(Mockito.anyString(), Mockito.anyString()); ExpandedSearchConverter.Results search = sut.search("joh", 0, 1); assertFalse(search.isOk()); - //Further calls fail too, token is stored + //Further calls fail too, token is stored (so getAccessToken is only resolved once) search = sut.search("joh", 0, 1); assertFalse(search.isOk()); diff --git a/dspace-api/src/test/resources/org/dspace/external/orcid-expanded-search.xml b/dspace-api/src/test/resources/org/dspace/external/orcid-expanded-search.xml new file mode 100644 index 000000000000..582731a42038 --- /dev/null +++ b/dspace-api/src/test/resources/org/dspace/external/orcid-expanded-search.xml @@ -0,0 +1,22 @@ + + + + + 0000-0002-9150-2529 + John + Connor + + + 0000-0002-1208-2352 + John + Kendrew + John Kendrew + + diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/AuthenticationRestControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/AuthenticationRestControllerIT.java index 63318926d752..b23811f27f17 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/AuthenticationRestControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/AuthenticationRestControllerIT.java @@ -71,6 +71,7 @@ import org.dspace.orcid.model.OrcidTokenResponseDTO; import org.dspace.services.ConfigurationService; import org.hamcrest.Matchers; +import org.junit.After; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -131,6 +132,45 @@ public class AuthenticationRestControllerIT extends AbstractControllerIntegratio private final String feature = CanChangePasswordFeature.NAME; + /** + * Configuration key for the ordered list of active AuthenticationMethod plugins. + */ + private static final String AUTH_PLUGIN_KEY = + "plugin.sequence.org.dspace.authenticate.AuthenticationMethod"; + + /** + * Replace the active AuthenticationMethod plugin sequence. + * + *

This sets the sequence via a JVM system property (plus an explicit + * {@link org.dspace.services.ConfigurationService#reloadConfig()} so the change is visible + * immediately) rather than via {@link org.dspace.services.ConfigurationService#setProperty(String, Object)}.

+ * + *

A plain {@code setProperty(...)} override only lives in the in-memory view of the combined + * configuration and is silently discarded whenever that view is rebuilt. The auto-reload listener + * rebuilds it as soon as any reloadable cfg file's last-modified timestamp changes (which happens + * intermittently during a CI run, e.g. another test writing {@code local.cfg}). When that rebuild lands + * between this call and the request under test, the on-disk default returns -- in CLARIN that default is + * {@code [PasswordAuthentication, ClarinShibAuthentication]} -- and a stray {@code password realm} leaks + * into the {@code WWW-Authenticate} header even though only e.g. Shibboleth was requested. A system + * property sits in the highest-precedence (override) section of the combined config and is re-read on + * every rebuild, so it survives auto-reload. It is cleared again in + * {@link #clearAuthenticationMethodSequence()}.

+ */ + private void setAuthenticationMethodSequence(String[] methods) { + System.setProperty(AUTH_PLUGIN_KEY, String.join(",", methods)); + configurationService.reloadConfig(); + } + + /** + * Remove the system-property override set by {@link #setAuthenticationMethodSequence(String[])} so it + * does not leak into other test classes running in the same JVM. Runs before the superclass @After, + * whose {@code reloadConfig()} then restores the on-disk default. + */ + @After + public void clearAuthenticationMethodSequence() { + System.clearProperty(AUTH_PLUGIN_KEY); + } + @Before public void setup() throws Exception { super.setUp(); @@ -140,7 +180,7 @@ public void setup() throws Exception { authorization = new Authorization(eperson, canChangePasswordFeature, ePersonRest); // Default all tests to Password Authentication only - configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", PASS_ONLY); + setAuthenticationMethodSequence(PASS_ONLY); } @Test @@ -198,7 +238,7 @@ public void testStatusGetSpecialGroups() throws Exception { .withName("specialGroupIP") .build(); - configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", PASS_AND_IP); + setAuthenticationMethodSequence(PASS_AND_IP); configurationService.setProperty("authentication-password.login.specialgroup","specialGroupPwd"); configurationService.setProperty("authentication-ip.specialGroupIP", "123.123.123.123"); context.restoreAuthSystemState(); @@ -338,7 +378,7 @@ public void testStatusNotAuthenticated() throws Exception { // @Test // public void testStatusShibAuthenticatedWithCookie() throws Exception { // //Enable Shibboleth login only -// configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", SHIB_ONLY); +// setAuthenticationMethodSequence(SHIB_ONLY); // // String uiURL = configurationService.getProperty("dspace.ui.url"); // @@ -458,7 +498,7 @@ public void testStatusNotAuthenticated() throws Exception { // @Test // public void testShibbolethEndpointCannotBeUsedWithShibDisabled() throws Exception { // // Enable only password login -// configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", PASS_ONLY); +// setAuthenticationMethodSequence(PASS_ONLY); // // String uiURL = configurationService.getProperty("dspace.ui.url"); // @@ -977,7 +1017,7 @@ public void testLoginGetRequest() throws Exception { public void testShibbolethLoginURLWithDefaultLazyURL() throws Exception { context.turnOffAuthorisationSystem(); //Enable Shibboleth login - configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", SHIB_ONLY); + setAuthenticationMethodSequence(SHIB_ONLY); //Create a reviewers group Group reviewersGroup = GroupBuilder.createGroup(context) @@ -1001,7 +1041,7 @@ public void testShibbolethLoginURLWithDefaultLazyURL() throws Exception { public void testShibbolethLoginURLWithServerURLContainingPort() throws Exception { context.turnOffAuthorisationSystem(); //Enable Shibboleth login - configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", SHIB_ONLY); + setAuthenticationMethodSequence(SHIB_ONLY); configurationService.setProperty("dspace.server.url", "http://localhost:8080/server"); configurationService.setProperty("authentication-shibboleth.lazysession.secure", false); @@ -1027,7 +1067,7 @@ public void testShibbolethLoginURLWithServerURLContainingPort() throws Exception public void testShibbolethLoginURLWithConfiguredLazyURL() throws Exception { context.turnOffAuthorisationSystem(); //Enable Shibboleth login - configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", SHIB_ONLY); + setAuthenticationMethodSequence(SHIB_ONLY); configurationService.setProperty("authentication-shibboleth.lazysession.loginurl", "http://shibboleth.org/Shibboleth.sso/Login"); @@ -1053,7 +1093,7 @@ public void testShibbolethLoginURLWithConfiguredLazyURL() throws Exception { public void testShibbolethLoginURLWithConfiguredLazyURLWithPort() throws Exception { context.turnOffAuthorisationSystem(); //Enable Shibboleth login - configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", SHIB_ONLY); + setAuthenticationMethodSequence(SHIB_ONLY); configurationService.setProperty("authentication-shibboleth.lazysession.loginurl", "http://shibboleth.org:8080/Shibboleth.sso/Login"); @@ -1081,7 +1121,7 @@ public void testShibbolethLoginURLWithConfiguredLazyURLWithPort() throws Excepti public void testShibbolethLoginRequestAttribute() throws Exception { context.turnOffAuthorisationSystem(); //Enable Shibboleth login - configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", SHIB_ONLY); + setAuthenticationMethodSequence(SHIB_ONLY); //Create a reviewers group Group reviewersGroup = GroupBuilder.createGroup(context) @@ -1137,7 +1177,7 @@ public void testShibbolethLoginRequestAttribute() throws Exception { @Ignore // Ignored until an endpoint is added to return all groups public void testShibbolethLoginRequestHeaderWithIpAuthentication() throws Exception { - configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", SHIB_AND_IP); + setAuthenticationMethodSequence(SHIB_AND_IP); configurationService.setProperty("authentication-ip.Administrator", "123.123.123.123"); @@ -1210,7 +1250,7 @@ public void testShibbolethLoginRequestHeaderWithIpAuthentication() throws Except @Test public void testShibbolethAndPasswordAuthentication() throws Exception { //Enable Shibboleth and password login - configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", SHIB_AND_PASS); + setAuthenticationMethodSequence(SHIB_AND_PASS); //Check if WWW-Authenticate header contains shibboleth and password getClient().perform(get("/api/authn/status").header("Referer", "http://my.uni.edu")) @@ -1281,7 +1321,7 @@ public void testShibbolethAndPasswordAuthentication() throws Exception { @Test public void testOnlyPasswordAuthenticationWorks() throws Exception { //Enable only password login - configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", PASS_ONLY); + setAuthenticationMethodSequence(PASS_ONLY); //Check if WWW-Authenticate header contains only getClient().perform(get("/api/authn/status").header("Referer", "http://my.uni.edu")) @@ -1314,7 +1354,7 @@ public void testOnlyPasswordAuthenticationWorks() throws Exception { @Test public void testShibbolethAuthenticationDoesNotWorkWithPassOnly() throws Exception { //Enable only password login - configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", PASS_ONLY); + setAuthenticationMethodSequence(PASS_ONLY); //Check if WWW-Authenticate header contains only password getClient().perform(get("/api/authn/status").header("Referer", "http://my.uni.edu")) @@ -1332,7 +1372,7 @@ public void testShibbolethAuthenticationDoesNotWorkWithPassOnly() throws Excepti @Test public void testOnlyShibbolethAuthenticationWorks() throws Exception { //Enable only Shibboleth login - configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", SHIB_ONLY); + setAuthenticationMethodSequence(SHIB_ONLY); //Check if WWW-Authenticate header contains only shibboleth getClient().perform(get("/api/authn/status").header("Referer", "http://my.uni.edu")) @@ -1365,7 +1405,7 @@ public void testOnlyShibbolethAuthenticationWorks() throws Exception { @Test public void testPasswordAuthenticationDoesNotWorkWithShibOnly() throws Exception { //Enable only Shibboleth login - configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", SHIB_ONLY); + setAuthenticationMethodSequence(SHIB_ONLY); getClient().perform(post("/api/authn/login") .param("user", eperson.getEmail()) @@ -1540,7 +1580,7 @@ public void testGenerateShortLivedTokenWithShortLivedToken() throws Exception { // @Test // public void testStatusOrcidAuthenticatedWithCookie() throws Exception { // -// configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", ORCID_ONLY); +// setAuthenticationMethodSequence(ORCID_ONLY); // // String uiURL = configurationService.getProperty("dspace.ui.url"); // @@ -1627,7 +1667,7 @@ public void testGenerateShortLivedTokenWithShortLivedToken() throws Exception { @Test public void testOrcidLoginURL() throws Exception { - configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", ORCID_ONLY); + setAuthenticationMethodSequence(ORCID_ONLY); String originalClientId = orcidConfiguration.getClientId(); orcidConfiguration.setClientId("CLIENT-ID"); @@ -1658,7 +1698,7 @@ public void testAreSpecialGroupsApplicable() throws Exception { .withName("specialGroupShib") .build(); - configurationService.setProperty("plugin.sequence.org.dspace.authenticate.AuthenticationMethod", SHIB_AND_PASS); + setAuthenticationMethodSequence(SHIB_AND_PASS); configurationService.setProperty("authentication-password.login.specialgroup", "specialGroupPwd"); configurationService.setProperty("authentication-shibboleth.role.faculty", "specialGroupShib"); configurationService.setProperty("authentication-shibboleth.default-roles", "faculty"); From 22cfef58e6209900e15d9a5501cd4779d8897b17 Mon Sep 17 00:00:00 2001 From: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> Date: Thu, 18 Jun 2026 15:37:05 +0200 Subject: [PATCH 30/41] Security patches from vanilla DSpace 7.6.7 (CVE-2026-49830, CVE-2026-49831) (#1340) * ORE aggregated resource URI validation (cherry picked from commit 7ac17f68432cff1e9463f05421fc8a29516000b8) * Velocity and template safety for Email and LDN messages * Safer Velocity configuration * New "message.templates.allowed-config" config * Remove "UnmodifiableConfiguration" in favour of a simple Map of whitelisted Config keys/values * Centralise Velocity config in core Utils * Small javadoc changes (cherry picked from commit b2d6141389f5652970b366325ed9deff21a86836) (cherry picked from commit 5b31db512f62bb530b71cb8fe85b1300f35e5601) * Better null checking in allowed config props (cherry picked from commit 6b665313cb48131ada04ae0840ff531b08b31dad) (cherry picked from commit 46a0dfb38197dfd9fa9970aa293e3a11a67c12b0) * Access configurationService at runtime, not rely on class setup (cherry picked from commit 5803819ba65e7211a4f49318d0d3bbf2246e21c1) (cherry picked from commit 4be430f4f0d404b88ad87454af8aeafefae9c042) * Remove strict mode Velocity engine configuration (allow nulls) (cherry picked from commit 655fc62874e9e4e5cf95ef2ff1e05b908484fc9f) * Filter requests for JSPs or traversal (cherry picked from commit cf9be8554d3597e2c80958cd62336b40b79ba19d) (cherry picked from commit dc3e4553641bdd91bac344d0402ad796b80a75f9) * Add additional logging to GlobalRequestSecurityFilter (cherry picked from commit 295a046fba14502619b3e3d96a7f6abdc9a4a5fc) (cherry picked from commit 0b1deae3fe94f55fb3dcda8dd6d6ba436b417f38) * Fix import order (cherry picked from commit e2e6a796fd8d19de18a80b735e20a62d29c3c5cd) (cherry picked from commit 2e400771353f2a6e50bfb0067e2c58ca602e04c9) * Update sitemap traversal test expectations (cherry picked from commit 56ae2871eaba764900e4d9e23685a9472f485069) (cherry picked from commit 1a3dfd7c1a341ce9c04ee1aef0bea9f45c24dcbc) * Backport GlobalRequestSecurityFilter for javax (cherry picked from commit 8a2eee9d4adcbb4d858252877df8b020426e0802) * Add secure file access methods (cherry picked from commit 22bec4459def712f529ff41283a8c7c5bcd1889c) * Backport Curation I/O using secure file access Removes some JDK >= 16 usage (cherry picked from commit 55905a2fc46b98194f92ec38e0fb8bafa7fee21a) * Curation config support for allowed base paths (cherry picked from commit 45022245be2fabb5ba26d50b335f1aa1f905a660) * Move curation -r reporter param to CLI only (cherry picked from commit 277af8233261ec0f61d71a4ce0908341c30e5e89) * Fix import order (cherry picked from commit a7572212c135155fb8420a2bfc95869f1ba6959d) * Ignore CurationScriptIT -T taskFile tests, to rewrite w/ CLI (cherry picked from commit 6437472b8277b9aa815dd71e14b499ba7515f87d) (cherry picked from commit 37cd6eb791d4f61bb54fedf899aa96f99504e38d) * Move taskfile -T option to CLI script config only (cherry picked from commit 00e4979a60fd69adbf4a7476926701ef59207ce7) (cherry picked from commit 27708ea6d70abe433f131cf3b875dfdc067d3c12) * UFAL/Allow lr.help.mail in email template config allowlist The 7.6.7 Velocity hardening restricts templates to an allowlisted "config" map (Utils.getAllowedTemplateConfig). UFAL/CLARIN templates (clarin_download_link_admin, clarin_token, matomo_report, share_submission) reference config.get('lr.help.mail'), which vanilla DSpace does not ship, so it was missing from the allowlist and those emails would render a null help address. Add it to dspace.cfg only; Utils.java stays identical to upstream. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Kim Shepherd Co-authored-by: Claude Opus 4.8 --- .../crosswalk/OREIngestionCrosswalk.java | 84 ++++++++- .../src/main/java/org/dspace/core/Email.java | 83 +++------ .../src/main/java/org/dspace/core/Utils.java | 71 ++++++++ .../main/java/org/dspace/curate/Curation.java | 32 +++- .../CurationCliScriptConfiguration.java | 4 + .../dspace/curate/CurationClientOptions.java | 7 +- .../storage/secure/SecureFileAccess.java | 166 ++++++++++++++++++ .../security/GlobalRequestSecurityFilter.java | 140 +++++++++++++++ .../app/rest/SitemapRestControllerIT.java | 12 +- .../org/dspace/curate/CurationScriptIT.java | 3 + dspace/config/dspace.cfg | 20 +++ dspace/config/modules/curate.cfg | 12 ++ dspace/config/modules/oai.cfg | 7 + 13 files changed, 560 insertions(+), 81 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/storage/secure/SecureFileAccess.java create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/security/GlobalRequestSecurityFilter.java diff --git a/dspace-api/src/main/java/org/dspace/content/crosswalk/OREIngestionCrosswalk.java b/dspace-api/src/main/java/org/dspace/content/crosswalk/OREIngestionCrosswalk.java index f756aae22577..9e890a6046fa 100644 --- a/dspace-api/src/main/java/org/dspace/content/crosswalk/OREIngestionCrosswalk.java +++ b/dspace-api/src/main/java/org/dspace/content/crosswalk/OREIngestionCrosswalk.java @@ -11,6 +11,8 @@ import java.io.IOException; import java.io.InputStream; import java.net.ConnectException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.sql.SQLException; import java.text.NumberFormat; @@ -18,6 +20,8 @@ import java.util.Date; import java.util.HashSet; import java.util.List; +import java.util.Locale; +import java.util.Objects; import java.util.Set; import org.apache.logging.log4j.Logger; @@ -34,6 +38,8 @@ import org.dspace.content.service.ItemService; import org.dspace.core.Constants; import org.dspace.core.Context; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; import org.jdom2.Attribute; import org.jdom2.Document; import org.jdom2.Element; @@ -76,6 +82,7 @@ public class OREIngestionCrosswalk .getBitstreamFormatService(); protected BundleService bundleService = ContentServiceFactory.getInstance().getBundleService(); protected ItemService itemService = ContentServiceFactory.getInstance().getItemService(); + protected ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); @Override @@ -173,9 +180,13 @@ public void ingest(Context context, DSpaceObject dso, Element root, boolean crea try { // Make sure the url string escapes all the oddball characters String processedURL = encodeForURL(href); - // Generate a requeset for the aggregated resource - ARurl = new URL(processedURL); - in = ARurl.openStream(); + if (validResourceUri(entryId, processedURL)) { + // Generate a request for the aggregated resource + ARurl = new URL(processedURL); + in = ARurl.openStream(); + } else { + throw new FileNotFoundException("Failed to validate " + processedURL); + } } catch (FileNotFoundException fe) { log.error("The provided URI failed to return a resource: " + href); } catch (ConnectException fe) { @@ -219,17 +230,17 @@ public void ingest(Context context, DSpaceObject dso, Element root, boolean crea * @param sourceString source unescaped string */ private String encodeForURL(String sourceString) { - Character lowalpha[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', + Character[] lowalpha = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; - Character upalpha[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', + Character[] upalpha = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'}; - Character digit[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}; - Character mark[] = {'-', '_', '.', '!', '~', '*', '\'', '(', ')'}; + Character[] digit = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}; + Character[] mark = {'-', '_', '.', '!', '~', '*', '\'', '(', ')'}; // reserved - Character reserved[] = {';', '/', '?', ':', '@', '&', '=', '+', '$', ',', '%', '#'}; + Character[] reserved = {';', '/', '?', ':', '@', '&', '=', '+', '$', ',', '%', '#'}; Set URLcharsSet = new HashSet(); URLcharsSet.addAll(Arrays.asList(lowalpha)); @@ -251,4 +262,61 @@ private String encodeForURL(String sourceString) { return processedString.toString(); } + /** + * Validate a resource URI against the host and scheme of the remote OAI endpoint, or a configured + * list of allowed prefixes. + * This still implicitly "trusts" the remote OAI server, but will reject resource URIs with a totally + * different hostname to avoid downloading malicious resources from a compromised endpoint. + * Even if the URL prefix validation is disabled, schemes will still be enforced to http(s) so file:/// and + * other unwanted schemes cannot be used + * @param entryUrl the entryId of the parent ORE resource + * @param resourceUrl the resource URL of the aggregated ORE resource + * @return result of the validation + */ + private boolean validResourceUri(String entryUrl, String resourceUrl) { + try { + Set allowedSchemes = Set.of("http", "https"); + URI entryUri = new URI(entryUrl).normalize(); + URI resourceUri = new URI(resourceUrl).normalize(); + String scheme = resourceUri.getScheme(); + + if (scheme == null || + !allowedSchemes.contains(scheme.toLowerCase(Locale.ROOT))) { + log.warn("Illegal scheme requested for ORE resource: {}", resourceUri); + return false; + } + + if (configurationService.getBooleanProperty("oai.harvester.ore.file.validateUrlPrefix", false)) { + for (String allowedPrefix : configurationService + .getArrayProperty("oai.harvester.ore.file.allowedUrlPrefix")) { + URI allowedUri = new URI(allowedPrefix).normalize(); + // Return true on the first allowed prefix match + if (Objects.equals(resourceUri.getScheme(), allowedUri.getScheme()) + && Objects.equals(resourceUri.getHost().toLowerCase(Locale.ROOT), + allowedUri.getHost().toLowerCase(Locale.ROOT))) { + return true; + } + } + + // If no allowed prefixes were matched, we require scheme + host to match the remote OAI server + if (!Objects.equals(entryUri.getScheme(), resourceUri.getScheme())) { + log.warn("Illegal scheme requested for ORE resource: {}", resourceUri); + return false; + } + if (!Objects.equals( + entryUri.getHost().toLowerCase(Locale.ROOT), + resourceUri.getHost().toLowerCase(Locale.ROOT))) { + log.warn("Illegal host requested for ORE resource: {}", resourceUri); + return false; + } + } + + return true; + + } catch (URISyntaxException e) { + log.warn("Could not validate ORE resource URI: {}", resourceUrl); + return false; + } + } + } diff --git a/dspace-api/src/main/java/org/dspace/core/Email.java b/dspace-api/src/main/java/org/dspace/core/Email.java index 98fcccae4c3c..0da9cb156694 100644 --- a/dspace-api/src/main/java/org/dspace/core/Email.java +++ b/dspace-api/src/main/java/org/dspace/core/Email.java @@ -22,7 +22,6 @@ import java.util.Date; import java.util.Enumeration; import java.util.List; -import java.util.Properties; import java.util.stream.Collectors; import javax.activation.DataHandler; import javax.activation.DataSource; @@ -45,12 +44,10 @@ import org.apache.logging.log4j.Logger; import org.apache.velocity.Template; import org.apache.velocity.VelocityContext; -import org.apache.velocity.app.Velocity; import org.apache.velocity.app.VelocityEngine; import org.apache.velocity.exception.MethodInvocationException; import org.apache.velocity.exception.ParseErrorException; import org.apache.velocity.exception.ResourceNotFoundException; -import org.apache.velocity.runtime.resource.loader.StringResourceLoader; import org.apache.velocity.runtime.resource.util.StringResourceRepository; import org.dspace.services.ConfigurationService; import org.dspace.services.factory.DSpaceServicesFactory; @@ -73,7 +70,7 @@ * Apache Velocity. They may contain VTL directives and property * placeholders. *

- * {@link addArgument(string)} adds a property to the {@code params} array + * {@link #addArgument(Object)} adds a property to the {@code params} array * in the Velocity context, which can be used to replace placeholder tokens * in the message. These arguments are indexed by number in the order they were * added to the message. @@ -81,9 +78,9 @@ * The DSpace configuration properties are also available to templates as the * array {@code config}, indexed by name. Example: {@code ${config.get('dspace.name')}} *

- * Recipients and attachments may be added as needed. See {@link addRecipient}, - * {@link addAttachment(File, String)}, and - * {@link addAttachment(InputStream, String, String)}. + * Recipients and attachments may be added as needed. See {@link #addRecipient}, + * {@link #addAttachment(File, String)}, and + * {@link #addAttachment(InputStream, String, String)}. *

* Headers such as Subject may be supplied by the template, by defining them * using the VTL directive {@code #set()}. Only headers named in the DSpace @@ -126,8 +123,8 @@ * *

* There are two ways to load a message body. One can create an instance of - * {@link Email} and call {@link setContent} on it, passing the body as a String. Or - * one can use the static factory method {@link getEmail} to load a file by its + * {@link Email} and call {@link #setContent} on it, passing the body as a String. Or + * one can use the static factory method {@link #getEmail} to load a file by its * complete filesystem path. In either case the text will be loaded into a * Velocity template. * @@ -173,18 +170,6 @@ public class Email { /** Velocity template settings. */ private static final String RESOURCE_REPOSITORY_NAME = "Email"; - private static final Properties VELOCITY_PROPERTIES = new Properties(); - static { - VELOCITY_PROPERTIES.put(Velocity.RESOURCE_LOADERS, "string"); - VELOCITY_PROPERTIES.put("resource.loader.string.description", - "Velocity StringResource loader"); - VELOCITY_PROPERTIES.put("resource.loader.string.class", - StringResourceLoader.class.getName()); - VELOCITY_PROPERTIES.put("resource.loader.string.repository.name", - RESOURCE_REPOSITORY_NAME); - VELOCITY_PROPERTIES.put("resource.loader.string.repository.static", - "false"); - } /** Velocity template for a message body */ private Template template; @@ -203,6 +188,13 @@ public Email() { charset = null; } + /** + * Get configuration service + */ + private static ConfigurationService getConfigurationService() { + return DSpaceServicesFactory.getInstance().getConfigurationService(); + } + /** * Add a recipient. * @@ -225,7 +217,7 @@ public void setContent(String name, String content) { arguments.clear(); VelocityEngine templateEngine = new VelocityEngine(); - templateEngine.init(VELOCITY_PROPERTIES); + templateEngine.init(Utils.getSecureVelocityProperties(RESOURCE_REPOSITORY_NAME)); StringResourceRepository repo = (StringResourceRepository) templateEngine.getApplicationAttribute(RESOURCE_REPOSITORY_NAME); @@ -255,7 +247,8 @@ public void setReplyTo(String email) { /** * Fill out the next argument in the template. * - * @param arg the value for the next argument + * @param arg the value for the next argument. If {@code null}, + * a zero-length string is substituted. */ public void addArgument(Object arg) { arguments.add(arg); @@ -333,7 +326,7 @@ public void reset() { * {@code mail.message.headers} then that name and its value will be added * to the message's headers. * - *

"subject" is treated specially: if {@link setSubject()} has not been + *

"subject" is treated specially: if {@link #setSubject} has not been * called, the value of any "subject" property will be used as if setSubject * had been called with that value. Thus a template may define its subject, * but the caller may override it. @@ -347,16 +340,13 @@ public void send() throws MessagingException, IOException { throw new MessagingException("Email has no body"); } - ConfigurationService config - = DSpaceServicesFactory.getInstance().getConfigurationService(); - // Get the mail configuration properties - String from = config.getProperty("mail.from.address"); - boolean disabled = config.getBooleanProperty("mail.server.disabled", false); + String from = getConfigurationService().getProperty("mail.from.address"); + boolean disabled = getConfigurationService().getBooleanProperty("mail.server.disabled", false); // If no character set specified, attempt to retrieve a default if (charset == null) { - charset = config.getProperty("mail.charset"); + charset = getConfigurationService().getProperty("mail.charset"); } // Get session @@ -371,11 +361,13 @@ public void send() throws MessagingException, IOException { new InternetAddress(recipient)); } // Get headers defined by the template. - String[] templateHeaders = config.getArrayProperty("mail.message.headers"); + String[] templateHeaders = getConfigurationService().getArrayProperty("mail.message.headers"); // Format the mail message body VelocityContext vctx = new VelocityContext(); - vctx.put("config", new UnmodifiableConfigurationService(config)); + // Pass a restricted (via configuration) list of resolved Configuration keys and values, for + // template lookup + vctx.put("config", Utils.getAllowedTemplateConfig()); vctx.put("params", Collections.unmodifiableList(arguments)); StringWriter writer = new StringWriter(); @@ -672,31 +664,4 @@ public OutputStream getOutputStream() throws IOException { throw new IOException("Cannot write to this read-only resource"); } } - - /** - * Wrap ConfigurationService to prevent templates from modifying - * the configuration. - */ - public static class UnmodifiableConfigurationService { - private final ConfigurationService configurationService; - - /** - * Swallow an instance of ConfigurationService. - * - * @param cs the real instance, to be wrapped. - */ - public UnmodifiableConfigurationService(ConfigurationService cs) { - configurationService = cs; - } - - /** - * Look up a key in the actual ConfigurationService. - * - * @param key to be looked up in the DSpace configuration. - * @return whatever value ConfigurationService associates with {@code key}. - */ - public String get(String key) { - return configurationService.getProperty(key); - } - } } diff --git a/dspace-api/src/main/java/org/dspace/core/Utils.java b/dspace-api/src/main/java/org/dspace/core/Utils.java index 047e0793713b..2ae8c679c910 100644 --- a/dspace-api/src/main/java/org/dspace/core/Utils.java +++ b/dspace-api/src/main/java/org/dspace/core/Utils.java @@ -30,16 +30,23 @@ import java.util.Collections; import java.util.Date; import java.util.GregorianCalendar; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; import java.util.Random; import java.util.StringTokenizer; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import com.coverity.security.Escape; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringSubstitutor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.velocity.app.Velocity; +import org.apache.velocity.runtime.resource.loader.StringResourceLoader; import org.dspace.services.ConfigurationService; import org.dspace.services.factory.DSpaceServicesFactory; import org.hibernate.Session; @@ -108,6 +115,11 @@ public final class Utils { private static final Calendar outCal = GregorianCalendar.getInstance(); + // Allowed configuration properties to pass to Velocity templates (Email, LDN) + private static final String[] DEFAULT_ALLOWED_TEMPLATE_CONFIGS = { + "dspace.name", "dspace.shortname", "dspace.ui.url", + "mail.helpdesk", "mail.message.helpdesk.telephone", "mail.admin", "mail.admin.name"}; + /** * Private constructor */ @@ -608,4 +620,63 @@ public static String fetchUUIDFromUrl(String urlString) { throw new IllegalArgumentException("Invalid URL or UUID format: " + e.getMessage(), e); } } + + /** + * Get a list of allowed DSpace configuration property keys that will be exposed to Velocity templates + * (used in Email and LDN messages) as a simple Map of strings. + * @return Map of strings representing resolved configuration properties + */ + public static Map getAllowedTemplateConfig() { + // Pass a restricted (via configuration) list of resolved Configuration keys and values, for + // template lookup + ConfigurationService configurationService = + DSpaceServicesFactory.getInstance().getConfigurationService(); + List allowedConfigurationKeys = List.of(configurationService.getArrayProperty( + "message.templates.allowed-config", DEFAULT_ALLOWED_TEMPLATE_CONFIGS)); + return allowedConfigurationKeys.stream() + .map(key -> { + String value = configurationService.getProperty(key); + return value != null ? Map.entry(key, value) : null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue + )); + } + + /** + * Create and return a set of default, secure Velocity configuration properties. + * @see {@link Email} + * + * @param resourceRepositoryName the templating context e.g. "LDN", "Email" + * @returns secure Velocity configuration for use with templating + */ + public static Properties getSecureVelocityProperties(String resourceRepositoryName) { + Properties secureVelocityProperties = new Properties(); + // Basic Velocity configuration + secureVelocityProperties.setProperty(Velocity.RESOURCE_LOADERS, "string"); + secureVelocityProperties.setProperty("resource.loader.string.description", + "Velocity StringResource loader"); + secureVelocityProperties.setProperty("resource.loader.string.class", + StringResourceLoader.class.getName()); + secureVelocityProperties.setProperty("resource.loader.string.repository.name", + resourceRepositoryName); + secureVelocityProperties.setProperty("resource.loader.string.repository.static", + "false"); + // Set secure default introspection and class restriction handling in Velocity + secureVelocityProperties.setProperty("introspector.uberspect.class", + "org.apache.velocity.util.introspection.SecureUberspector"); + secureVelocityProperties.setProperty("introspector.restrict.classes", + "java.lang.Class,java.lang.Runtime,java.lang.System"); + secureVelocityProperties.setProperty( "introspector.restrict.packages", + "java.lang.reflect,java.io,java.nio"); + // Set strict mode if configured (default: false, as we've always treated null values as blanks) + if (DSpaceServicesFactory.getInstance().getConfigurationService() + .getBooleanProperty("message.templates.strict_mode", false)) { + secureVelocityProperties.setProperty("runtime.strict_mode.enable", "true"); + } + + return secureVelocityProperties; + } } diff --git a/dspace-api/src/main/java/org/dspace/curate/Curation.java b/dspace-api/src/main/java/org/dspace/curate/Curation.java index d828e3e714d8..091ce00bcba2 100644 --- a/dspace-api/src/main/java/org/dspace/curate/Curation.java +++ b/dspace-api/src/main/java/org/dspace/curate/Curation.java @@ -10,11 +10,15 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileNotFoundException; -import java.io.FileReader; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.UUID; @@ -36,6 +40,7 @@ import org.dspace.handle.service.HandleService; import org.dspace.scripts.DSpaceRunnable; import org.dspace.services.factory.DSpaceServicesFactory; +import org.dspace.storage.secure.SecureFileAccess; import org.dspace.utils.DSpace; /** @@ -111,8 +116,16 @@ private void handleCurationTask(Curator curator) throws IOException, SQLExceptio } else if (commandLine.hasOption('T')) { // load taskFile BufferedReader reader = null; + // in this case, Curation CLI expects to calculate the -T parameter from the user's current working dir + String taskFilePath = SecureFileAccess.calculateAbsolutePathUsingCwd(this.taskFile); try { - reader = new BufferedReader(new FileReader(this.taskFile)); + String dspaceDir = DSpaceServicesFactory.getInstance() + .getConfigurationService().getProperty("dspace.dir"); + List allowedTaskFileBasePath = new ArrayList<>( + Arrays.asList(DSpaceServicesFactory.getInstance().getConfigurationService() + .getArrayProperty("curate.taskfile.base", new String[]{dspaceDir}))); + reader = SecureFileAccess.getBufferedReader(taskFilePath, allowedTaskFileBasePath, + "curation-taskfile", StandardCharsets.UTF_8); while ((taskName = reader.readLine()) != null) { if (verbose) { super.handler.logInfo("Adding task: " + taskName); @@ -197,12 +210,25 @@ private void endScript(long timeRun) throws SQLException { */ private Curator initCurator() throws FileNotFoundException { Curator curator = new Curator(handler); + String dspaceDir = DSpaceServicesFactory.getInstance() + .getConfigurationService().getProperty("dspace.dir"); + List allowedReporterBasePaths = new ArrayList<>(Arrays.asList(DSpaceServicesFactory.getInstance() + .getConfigurationService().getArrayProperty("curate.reporter.base", + new String[]{dspaceDir + File.separatorChar + "log"}))); if (null == this.reporter) { outputReporter = new DoNothingReporter(); } else if ("-".equals(this.reporter)) { outputReporter = new SystemOutReporter(); } else { - outputReporter = new FilePrinterReporter(this.reporter); + // Reporter param comes from CLI execution. Calculate abs path from user's current working dir + String reporterFilePath = SecureFileAccess.calculateAbsolutePathUsingCwd(this.reporter); + try { + Path validatedReporterPath = SecureFileAccess.validatePathForWrite( + reporterFilePath, allowedReporterBasePaths, "curation-reporter"); + outputReporter = new FilePrinterReporter(validatedReporterPath.toString()); + } catch (IOException e) { + throw new FileNotFoundException(e.getLocalizedMessage()); + } } curator.setReporter(outputReporter); diff --git a/dspace-api/src/main/java/org/dspace/curate/CurationCliScriptConfiguration.java b/dspace-api/src/main/java/org/dspace/curate/CurationCliScriptConfiguration.java index eaa04f477829..925bd4f2d232 100644 --- a/dspace-api/src/main/java/org/dspace/curate/CurationCliScriptConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/curate/CurationCliScriptConfiguration.java @@ -20,6 +20,10 @@ public Options getOptions() { options = super.getOptions(); options.addOption("e", "eperson", true, "email address of curating eperson"); options.getOption("e").setRequired(true); + options.addOption("r", "reporter", true, + "relative or absolute path to the desired report file. Use '-' to report to console. If absent, no " + + "reporting"); + options.addOption("T", "taskfile", true, "file containing curation task names"); return options; } } diff --git a/dspace-api/src/main/java/org/dspace/curate/CurationClientOptions.java b/dspace-api/src/main/java/org/dspace/curate/CurationClientOptions.java index 8ec0f14697c0..03ad2f34b230 100644 --- a/dspace-api/src/main/java/org/dspace/curate/CurationClientOptions.java +++ b/dspace-api/src/main/java/org/dspace/curate/CurationClientOptions.java @@ -31,7 +31,8 @@ public enum CurationClientOptions { /** * This method resolves the CommandLine parameters to figure out which action the curation script should perform * - * @param commandLine The relevant CommandLine for the curation script + * @param commandLine The relevant CommandLine for the curation script. Note that -T is passed only + * from CurationCliScriptConfig and is not accessible from UI processes * @return The curation option to be ran, parsed from the CommandLine */ protected static CurationClientOptions getClientOption(CommandLine commandLine) { @@ -54,14 +55,10 @@ protected static Options constructOptions() { Options options = new Options(); options.addOption("t", "task", true, "curation task name; options: " + getTaskOptions()); - options.addOption("T", "taskfile", true, "file containing curation task names"); options.addOption("i", "id", true, "Id (handle) of object to perform task on, or 'all' to perform on whole repository"); options.addOption("p", "parameter", true, "a task parameter 'NAME=VALUE'"); options.addOption("q", "queue", true, "name of task queue to process"); - options.addOption("r", "reporter", true, - "relative or absolute path to the desired report file. Use '-' to report to console. If absent, no " + - "reporting"); options.addOption("s", "scope", true, "transaction scope to impose: use 'object', 'curation', or 'open'. If absent, 'open' applies"); options.addOption("v", "verbose", false, "report activity to stdout"); diff --git a/dspace-api/src/main/java/org/dspace/storage/secure/SecureFileAccess.java b/dspace-api/src/main/java/org/dspace/storage/secure/SecureFileAccess.java new file mode 100644 index 000000000000..45727d9ee5ae --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/storage/secure/SecureFileAccess.java @@ -0,0 +1,166 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.storage.secure; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +/** + * Decent I/O path validation - not perfect when symlinks are used and we are writing + * as 'toRealPath' check on the resolved path fails for new files + * + * @author Kim Shepherd + */ +public final class SecureFileAccess { + + private SecureFileAccess() {} + + /** + * Validate a given path against an allowed base path. Does not attempt to calculate "real path" + * before validation, as this breaks for new files which don't yet exist. This can make the resulting + * validation still vulnerable to symlink traversal in some cases + * @param file the unvalidated file, usually derived from user input or configuration + * This MUST be an absolute path, and the caller is expected to calculate it based on best + * context (e.g. configured base path, CWD, dspace.dir, and so on) + * @param allowedBasePaths list of allowed base paths for this use case as per system configuration + * @param purpose the name of the calling component / use case for logging and inspection + * @throws IOException on validation failure + */ + public static Path validatePathForWrite(String file, List allowedBasePaths, String purpose) + throws IOException { + Path filePath = Path.of(file); + if (!filePath.isAbsolute()) { + throw new IOException("Absolute path required for I/O (" + purpose + "): " + file); + } + for (String allowedBasePath : allowedBasePaths) { + Path basePath = Path.of(allowedBasePath) + .toRealPath() + .normalize(); + Path resolvedPath = basePath.resolve(file).normalize(); + if (resolvedPath.startsWith(basePath)) { + return resolvedPath; + } + } + + // If no valid path was resolved and returned by now + // we raise an exception and treat this as illegal access + throw new IOException("Illegal file path attempted for I/O (" + purpose + "): " + file); + } + + /** + * Validate a given path against an allowed base path. + * More secure than the 'write' variant because we can explicitly resolve links as well. + * + * @param file the unvalidated file, usually derived from user input or configuration + * This MUST be an absolute path, and the caller is expected to calculate it based on best + * context (e.g. configured base path, CWD, dspace.dir, and so on) + * @param allowedBasePaths the allowed base paths for this use case as per system configuration + * @param purpose the name of the calling component / use case for logging and inspection + * @throws IOException on validation failure + */ + public static Path validatePathForRead(String file, List allowedBasePaths, String purpose) + throws IOException { + Path filePath = Path.of(file); + if (!filePath.isAbsolute()) { + throw new IOException("Absolute path required for I/O (" + purpose + "): " + file); + } + for (String allowedBasePath : allowedBasePaths) { + Path basePath = Path.of(allowedBasePath) + .toRealPath() + .normalize(); + Path resolvedPath = basePath.resolve(file).toRealPath().normalize(); + if (resolvedPath.startsWith(basePath)) { + return resolvedPath; + } + } + // If no valid path was resolved and returned by now + // we raise an exception and treat this as illegal access + throw new IOException("Illegal file path attempted for I/O (" + purpose + "): " + file); + } + + /** + * Get a buffered reader after validating file path. + * @param unvalidatedFile the unvalidated file, usually derived from user input or configuration + * @param allowedBasePaths the allowed base paths for this use case as per system configuration + * @param purpose the name of the calling component / use case for logging and inspection + * @throws IOException on validation failure + */ + public static BufferedReader getBufferedReader(String unvalidatedFile, List allowedBasePaths, + String purpose, Charset charset) throws IOException { + if (charset == null) { + charset = StandardCharsets.UTF_8; + } + Path validatedFile = validatePathForRead(unvalidatedFile, allowedBasePaths, purpose); + return Files.newBufferedReader(validatedFile, charset); + } + + /** + * Get an input stream after validating file path. + * @param unvalidatedFile the unvalidated file, usually derived from user input or configuration + * @param allowedBasePaths the allowed base paths for this use case as per system configuration + * @param purpose the name of the calling component / use case for logging and inspection + * @throws IOException on validation failure + */ + public static InputStream getInputStream(String unvalidatedFile, List allowedBasePaths, String purpose) + throws IOException { + Path validatedFile = validatePathForRead(unvalidatedFile, allowedBasePaths, purpose); + return Files.newInputStream(validatedFile); + + } + + /** + * Get an output stream after validating file path. New files can't use toRealPath() for link calculation so + * there is a bit of a trade-off in allowing some symlink traversal to occur + * @param unvalidatedFile the unvalidated file, usually derived from user input or configuration + * @param allowedBasePaths the allowed base paths for this use case as per system configuration + * @param purpose the name of the calling component / use case for logging and inspection + * @throws IOException on validation failure + */ + public static OutputStream getOutputStream(String unvalidatedFile, List allowedBasePaths, String purpose) + throws IOException { + Path validatedFile = validatePathForWrite(unvalidatedFile, allowedBasePaths, purpose); + return Files.newOutputStream(validatedFile); + } + + /** + * Calculate an absolute path (if not already absolute) using current working dir as a root + * for relative file paths + * @param file the relative or absolute file given as input + * @return absolute path calculated from file and cwd + */ + public static String calculateAbsolutePathUsingCwd(String file) { + String filePath = file; + Path path = Path.of(filePath); + if (!path.isAbsolute()) { + filePath = Path.of("").toAbsolutePath().resolve(path).normalize().toString(); + } + return filePath; + } + + /** + * Calculate an absolute path (if not already absolute) using a given base dir as a root + * for relative file paths + * @param file the relative or absolute file given as input + * @return absolute path calculated from file and base dir + */ + public static String calculateAbsolutePathUsingBaseDir(String file, String baseDir) { + String filePath = file; + Path path = Path.of(filePath); + if (!path.isAbsolute()) { + filePath = Path.of(baseDir).toAbsolutePath().resolve(path).normalize().toString(); + } + return filePath; + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/security/GlobalRequestSecurityFilter.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/security/GlobalRequestSecurityFilter.java new file mode 100644 index 000000000000..a9f0eea8d728 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/security/GlobalRequestSecurityFilter.java @@ -0,0 +1,140 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.security; + +import java.io.IOException; +import java.net.URI; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.Locale; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +/** + * Global filter acting on all requests (not just /api/) to provide some additional hardening + * against common attacks or RCE, if a malicious payload was somehow written to a directory + * executable by the servlet container. + * The decoding and normalisation is designed to be tolerant of malformed URLs or broken clients, etc. + * so that this additional security filter does not introduce false positives or unintended side effects. + * + * @author Kim Shepherd + */ +@Component +@Order(Ordered.HIGHEST_PRECEDENCE) +public class GlobalRequestSecurityFilter extends OncePerRequestFilter { + + @Override + protected void doFilterInternal( + HttpServletRequest request, + HttpServletResponse response, + FilterChain filterChain + ) throws ServletException, IOException { + String normalizedPath = normaliseUrl(request.getRequestURI()); + // Return 403 forbidden if JSP execution or URL traversal is attempted + if (isTraversalAttempt(normalizedPath)) { + logger.warn("Path traversal attempt detected. Skipping request: " + request.getRequestURI()); + response.sendError(HttpServletResponse.SC_FORBIDDEN); + return; + } + if (isJspExecutionAttempt(normalizedPath)) { + logger.warn("JSP execution attempt detected. Skipping request: " + request.getRequestURI()); + response.sendError(HttpServletResponse.SC_FORBIDDEN); + return; + } + filterChain.doFilter(request, response); + } + + /** + * Normalise the URI similarly to Tomcat, for testing how it will be interpreted + * @param rawUrl the unvalidated URL string + * @return a decoded, normalise URL + */ + private String normaliseUrl(String rawUrl) throws IOException { + if (rawUrl == null || rawUrl.isBlank()) { + throw new IOException("Empty URL"); + } + String url = rawUrl.split("\\?")[0]; + // Strip ;jspsession=... and so on + int semicolon = url.indexOf(';'); + if (semicolon >= 0) { + url = url.substring(0, semicolon); + } + url = decodeUrl(url); + if (url == null || url.isBlank()) { + throw new IOException("Decoded URL path is empty"); + } + url = normaliseUrlPath(url); + if (url == null || url.isBlank()) { + throw new IOException("Normalised URL path is empty"); + } + return url.toLowerCase(Locale.ROOT); + } + + /** + * Decode URL, falling back to original URL if it's malformed or undecodable + * @param url the encoded / unvalidated URL + * @return decoded URL or the original URL on error + */ + private String decodeUrl(String url) { + try { + return URLDecoder.decode(url, StandardCharsets.UTF_8); + } catch (IllegalArgumentException ex) { + // if we can't decode it, just return raw string + return url; + } + } + + /** + * Normalise the URL path and ensure it ends in a / + * @param url the URL path to normalise + * @return normalised path or the original parameter on error + */ + private String normaliseUrlPath(String url) { + try { + if (!url.startsWith("/")) { + url = "/" + url; + } + return new URI(url).normalize().getPath(); + } catch (Exception e) { + // if we can't use or normalise the path, just return the raw string + return url; + } + } + + /** + * Detect traversal after normalisation + * @param url the URL path to validate + * @return true if this looks like a traversal attempt + */ + private boolean isTraversalAttempt(String url) { + return url.contains("../") + || url.contains("/..") + || url.contains("%2e%2e") + || url.contains(".."); + } + + /** + * Block JSP execution attempts + * @param url the URL path to validate + */ + private boolean isJspExecutionAttempt(String url) { + return url.endsWith(".jsp") + || url.endsWith(".jspx") + || url.contains(".jsp/") + || url.contains(".jspx/") + || url.contains(".jsp\0") + || url.contains(".jspx\0"); + } +} diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java index 04d22718e846..084b8272bd09 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/SitemapRestControllerIT.java @@ -14,8 +14,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import javax.servlet.ServletException; - import org.dspace.app.rest.test.AbstractControllerIntegrationTest; import org.dspace.authorize.service.ResourcePolicyService; import org.dspace.builder.CollectionBuilder; @@ -131,18 +129,20 @@ public void testSitemap_notValidSiteMapFile() throws Exception { .andExpect(status().isNotFound()); } - @Test(expected = ServletException.class) + @Test public void testSitemap_fileSystemTraversal_dspaceCfg() throws Exception { //** WHEN ** //We attempt to use endpoint for malicious file system traversal - getClient().perform(get("/" + SITEMAPS_ENDPOINT + "/%2e%2e/config/dspace.cfg")); + getClient().perform(get("/" + SITEMAPS_ENDPOINT + "/%2e%2e/config/dspace.cfg")) + .andExpect(status().isForbidden()); } - @Test(expected = ServletException.class) + @Test public void testSitemap_fileSystemTraversal_dspaceCfg2() throws Exception { //** WHEN ** //We attempt to use endpoint for malicious file system traversal - getClient().perform(get("/" + SITEMAPS_ENDPOINT + "/%2e%2e%2fconfig%2fdspace.cfg")); + getClient().perform(get("/" + SITEMAPS_ENDPOINT + "/%2e%2e%2fconfig%2fdspace.cfg")) + .andExpect(status().isForbidden()); } @Test diff --git a/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java b/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java index 8745613d7af6..347ed0935f05 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/curate/CurationScriptIT.java @@ -49,6 +49,7 @@ import org.dspace.scripts.configuration.ScriptConfiguration; import org.dspace.scripts.factory.ScriptServiceFactory; import org.dspace.scripts.service.ScriptService; +import org.junit.Ignore; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -214,6 +215,7 @@ public void curateScript_InvalidScope() throws Exception { .andExpect(status().isBadRequest()); } + @Ignore @Test public void curateScript_InvalidTaskFile() throws Exception { String token = getAuthToken(admin.getEmail(), password); @@ -286,6 +288,7 @@ public void curateScript_validRequest_Task() throws Exception { } } + @Ignore @Test public void curateScript_validRequest_TaskFile() throws Exception { context.turnOffAuthorisationSystem(); diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index 8330d7ee3cf1..add4564d547b 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -162,6 +162,7 @@ mail.from.address = dspace-noreply@myu.edu # will use the above settings to create a Session. #mail.session.name = Session + # When feedback is submitted via the Feedback form, it is sent to this address # Currently limited to one recipient! # if this property is empty or commented out, feedback form is disabled @@ -228,6 +229,25 @@ mail.message.headers = charset # Helpdesk telephone. Not email, but should be with other contact info. Optional. #mail.message.helpdesk.telephone = +1 555 555 5555 +# Allowed configuration properties, to pass in a "config" map to email and LDN templates. +# This allows templates to easily access dynamic configuration properties, without +# exposing sensitive information to the templating engine +message.templates.allowed-config = dspace.name +message.templates.allowed-config = dspace.shortname +message.templates.allowed-config = dspace.ui.url +message.templates.allowed-config = mail.helpdesk +message.templates.allowed-config = mail.message.helpdesk.telephone +message.templates.allowed-config = mail.admin +message.templates.allowed-config = mail.admin.name +# UFAL/CLARIN email templates (clarin_download_link_admin, clarin_token, matomo_report, share_submission) +# reference this key; required since the template config is now allowlisted +message.templates.allowed-config = lr.help.mail + +# Whether to run Velocity in strict mode (null parameter values in templates for LDN or Email will result +# in an Exception instead of a blank string) +# Default: false (this can introduce unwanted side-effects if e.g. a submitter eperson is deleted for a workflow task) +#message.templates.strict_mode = false + ##### Asset Storage (bitstreams / files) ###### # Moved to config/spring/api/bitstore.xml diff --git a/dspace/config/modules/curate.cfg b/dspace/config/modules/curate.cfg index 62e6e3644d8f..35b60c63d658 100644 --- a/dspace/config/modules/curate.cfg +++ b/dspace/config/modules/curate.cfg @@ -31,3 +31,15 @@ curate.taskqueue.dir = ${dspace.dir}/ctqueues # Maximum amount of redirects set to 0 for none and -1 for unlimited curate.checklinks.max-redirect = 0 + +# allowed base path(s) of curation task files +# it is recommended to restrict this path as much as possible +# so that the DSpace Processes framework may only load files as "tasks" +# from a trusted location. For multiple paths, repeat this configuration +# property for each trusted path +# Default: ${dspace.dir} +#curate.taskfile.base = ${dspace.dir} + +# allowed base path of reporter output. +# Default: ${dspace.dir}/log +#curate.reporter.base = ${dspace.dir}/log diff --git a/dspace/config/modules/oai.cfg b/dspace/config/modules/oai.cfg index 8d9d9b1ae219..a35990693ffb 100644 --- a/dspace/config/modules/oai.cfg +++ b/dspace/config/modules/oai.cfg @@ -156,3 +156,10 @@ oai.harvester.unknownSchema = fail # when attempting to find the handle of harvested items. If there is a match with # this config parameter, a new handle will be minted instead. Default value: 123456789. #oai.harvester.rejectedHandlePrefix = 123456789, myTestHandle + +# If ingesting files with ORE, only files with URLs that match the base URL of the remote +# OAI endpoint's domain name are accepted, or a list of other URL prefixes defined below +#oai.harvester.ore.file.validateUrlPrefix = true +# Prefixes that are allowed globally (for any endpoint) are below +#oai.harvester.ore.file.allowedUrlPrefix = dspace.myinstitution.edu +#oai.harvester.ore.file.allowedUrlPrefix = files.myinstitution.edu From 882904ba3537387653c3778de454a851d11b7efe Mon Sep 17 00:00:00 2001 From: Kasinhou <129340513+Kasinhou@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:47:13 +0200 Subject: [PATCH 31/41] Autolabel for new issues (#1341) Co-authored-by: Matus Kasak --- .github/workflows/new_issue_label.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/new_issue_label.yml diff --git a/.github/workflows/new_issue_label.yml b/.github/workflows/new_issue_label.yml new file mode 100644 index 000000000000..e9fcf70ae6c9 --- /dev/null +++ b/.github/workflows/new_issue_label.yml @@ -0,0 +1,18 @@ +name: Auto Label New Issues + +on: + issues: + types: [opened] + +jobs: + label-issue: + runs-on: ubuntu-latest + permissions: + issues: write + + steps: + - name: Auto Label + uses: dataquest-dev/gh-actions/start@main + + with: + github-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From 15b296aa2a8ec086df03a9194c52a725ab1ba91e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Tue, 23 Jun 2026 11:09:29 +0200 Subject: [PATCH 32/41] [Port to dtq-dev] Issue 1364: tgz file preview fix (#1338) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Issue 1364: tgz file preview fix (ufal/clarin-dspace#1372) * Issue 1364: tgz file preview fix * resolve MR comments + fixed test * Update help message for force preview option * get rid of the password requirement on FilePreview script --------- Co-authored-by: Ondřej Košarko (cherry picked from commit 00a2a37b89b700bf14c51c0dc8019f31d66c54e4) * PR Comments --------- Co-authored-by: Milan Kuchtiak --- .../content/PreviewContentServiceImpl.java | 2 +- .../scripts/filepreview/FilePreview.java | 64 +++++++-------- .../filepreview/FilePreviewConfiguration.java | 10 +-- .../scripts/filepreview/FilePreviewIT.java | 77 +++++++++++++------ .../app/rest/PreviewContentServiceImplIT.java | 26 ++++++- 5 files changed, 114 insertions(+), 65 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/PreviewContentServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/PreviewContentServiceImpl.java index a29e05e8aefe..0abdf603b631 100644 --- a/dspace-api/src/main/java/org/dspace/content/PreviewContentServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/PreviewContentServiceImpl.java @@ -369,7 +369,7 @@ private void processGzipFile(List filePaths, File file, Bitstream bitstr if (fileName == null) { logBitstreamNameIsNull(); } else { - if (fileName.toLowerCase().endsWith("tar.gz")) { + if (fileName.toLowerCase().endsWith(".tar.gz") || fileName.toLowerCase().endsWith(".tgz")) { processTarGzipFile(filePaths, file, bitstream); } else { try (InputStream is = new GzipCompressorInputStream(new FileInputStream(file))) { diff --git a/dspace-api/src/main/java/org/dspace/scripts/filepreview/FilePreview.java b/dspace-api/src/main/java/org/dspace/scripts/filepreview/FilePreview.java index ba0b0ef0bf12..38e10361e9f9 100644 --- a/dspace-api/src/main/java/org/dspace/scripts/filepreview/FilePreview.java +++ b/dspace-api/src/main/java/org/dspace/scripts/filepreview/FilePreview.java @@ -16,12 +16,10 @@ import org.apache.commons.cli.ParseException; import org.apache.commons.lang3.StringUtils; -import org.dspace.authenticate.AuthenticationMethod; -import org.dspace.authenticate.factory.AuthenticateServiceFactory; -import org.dspace.authenticate.service.AuthenticationService; import org.dspace.content.Bitstream; import org.dspace.content.Bundle; import org.dspace.content.Item; +import org.dspace.content.PreviewContent; import org.dspace.content.factory.ContentServiceFactory; import org.dspace.content.service.ItemService; import org.dspace.content.service.PreviewContentService; @@ -46,13 +44,12 @@ public class FilePreview extends DSpaceRunnable { ContentServiceFactory.getInstance().getPreviewContentService(); private EPersonService ePersonService = EPersonServiceFactory.getInstance() .getEPersonService(); - private AuthenticationService authenticateService = AuthenticateServiceFactory.getInstance() - .getAuthenticationService(); /** * `-i`: Info, show help information. */ private boolean info = false; + private boolean force = false; /** * `-u`: UUID of the Item for which to create a preview of its bitstreams. @@ -60,7 +57,6 @@ public class FilePreview extends DSpaceRunnable { private String specificItemUUID = null; private String epersonMail = null; - private String epersonPassword = null; @Override public FilePreviewConfiguration getScriptConfiguration() { @@ -84,11 +80,14 @@ public void setup() throws ParseException { specificItemUUID); } + if (commandLine.hasOption('f')) { + force = true; + } + epersonMail = commandLine.getOptionValue('e'); - epersonPassword = commandLine.getOptionValue('p'); - if (getEpersonIdentifier() == null && (epersonMail == null || epersonPassword == null)) { - throw new ParseException("Provide both -e/--email and -p/--password when no eperson is supplied."); + if (getEpersonIdentifier() == null && epersonMail == null) { + throw new ParseException("Provide -e/--email when no eperson is supplied."); } } @@ -101,7 +100,7 @@ public void internalRun() throws Exception { Context context = new Context(); try { - context.setCurrentUser(getAuthenticatedEperson((context))); + context.setCurrentUser(getEperson(context)); handler.logInfo("Authentication by user: " + context.getCurrentUser().getEmail()); if (StringUtils.isNotBlank(specificItemUUID)) { // Generate the preview only for a specific item @@ -152,7 +151,17 @@ private void generateItemFilePreviews(Context context, UUID itemUUID) throws Exc } // Generate new content if we didn't find any if (previewContentService.hasPreview(context, bitstream)) { - continue; + if (force) { + List previewContents = previewContentService + .findByBitstream(context, bitstream.getID()); + for (PreviewContent content : previewContents) { + handler.logInfo("Deleting existing preview content: '" + content.getName() + + "', for bitstream: '" + bitstream.getName() + "'"); + previewContentService.delete(context, content); + } + } else { + continue; + } } List fileInfos = previewContentService.getFilePreviewContent(context, bitstream); @@ -162,6 +171,7 @@ private void generateItemFilePreviews(Context context, UUID itemUUID) throws Exc continue; } + handler.logInfo("Generating file preview for bitstream: " + bitstream.getName()); for (FileInfo fi : fileInfos) { previewContentService.createPreviewContent(context, bitstream, fi); } @@ -176,38 +186,30 @@ public void printHelp() { "You can choose from these available options:\n" + " -i, --info Show help information\n" + " -u, --uuid The UUID of the ITEM for which to create a preview of its bitstreams\n" + - " -e, --email Email for authentication\n" + - " -p, --password Password for authentication\n"); + " -f, --force Force to create preview, even when the preview exists\n" + + " -e, --email Email of the eperson to run the script as\n"); } /** - * Retrieves an EPerson object either by its identifier or by performing an email-based lookup. - * It then authenticates the EPerson using the provided email and password. - * If the authentication is successful, it returns the EPerson object; otherwise, - * it throws an AuthenticationException. + * Resolves the EPerson the script runs as: the eperson supplied by the launching context + * (e.g. the logged-in user when started from the admin UI) if present, otherwise the eperson + * looked up by the {@code -e}/--email option. Like other CLI scripts, command-line invocation + * is trusted (shell access implies full server access), so no password is verified here; + * admin-only operations remain guarded by authorization checks in the service layer. * * @param context The Context object used for interacting with the DSpace database and service layer. - * @return The authenticated EPerson object corresponding to the provided email, - * if authentication is successful. - * @throws SQLException If a database error occurs while retrieving or interacting with the EPerson data. - * @throws AuthenticationException If no EPerson is found for the provided email - * or if the authentication fails. + * @return The EPerson the script should run as. + * @throws SQLException If a database error occurs while retrieving the EPerson data. + * @throws AuthenticationException If no EPerson is found for the provided email. */ - private EPerson getAuthenticatedEperson(Context context) throws SQLException, AuthenticationException { + private EPerson getEperson(Context context) throws SQLException, AuthenticationException { if (getEpersonIdentifier() != null) { return ePersonService.find(context, getEpersonIdentifier()); } - String msg; EPerson ePerson = ePersonService.findByEmail(context, epersonMail); if (ePerson == null) { - msg = "No EPerson found for this email: " + epersonMail; - handler.logError(msg); - throw new AuthenticationException(msg); - } - int authenticated = authenticateService.authenticate(context, epersonMail, epersonPassword, null, null); - if (AuthenticationMethod.SUCCESS != authenticated) { - msg = "Authentication failed for email: " + epersonMail; + String msg = "No EPerson found for this email: " + epersonMail; handler.logError(msg); throw new AuthenticationException(msg); } diff --git a/dspace-api/src/main/java/org/dspace/scripts/filepreview/FilePreviewConfiguration.java b/dspace-api/src/main/java/org/dspace/scripts/filepreview/FilePreviewConfiguration.java index 610691434060..a46b4fbe4eff 100644 --- a/dspace-api/src/main/java/org/dspace/scripts/filepreview/FilePreviewConfiguration.java +++ b/dspace-api/src/main/java/org/dspace/scripts/filepreview/FilePreviewConfiguration.java @@ -39,15 +39,11 @@ public Options getOptions() { options.getOption("u").setType(String.class); options.getOption("u").setRequired(false); + options.addOption("f", "force", false, "Force to create preview, even when the preview exists."); + options.addOption("e", "email", true, - "Email for authentication."); + "Email of the eperson to run the script as."); options.getOption("e").setType(String.class); - options.getOption("e").setRequired(true); - - options.addOption("p", "password", true, - "Password for authentication."); - options.getOption("p").setType(String.class); - options.getOption("p").setRequired(true); super.options = options; } diff --git a/dspace-api/src/test/java/org/dspace/scripts/filepreview/FilePreviewIT.java b/dspace-api/src/test/java/org/dspace/scripts/filepreview/FilePreviewIT.java index aeb8e050e9ef..49bf48d49711 100644 --- a/dspace-api/src/test/java/org/dspace/scripts/filepreview/FilePreviewIT.java +++ b/dspace-api/src/test/java/org/dspace/scripts/filepreview/FilePreviewIT.java @@ -104,21 +104,11 @@ public void testUnauthorizedEmail() throws Exception { assertEquals(1, run); // Since a ParseException was caught, expect return code 1 } - @Test - public void testUnauthorizedPassword() throws Exception { - // Run the script - TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "file-preview", "-e", ePerson.getEmail()}; - int run = ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), - testDSpaceRunnableHandler, kernelImpl); - assertEquals(1, run); // Since a ParseException was caught, expect return code 1 - } - @Test public void testWhenNoFilesRun() throws Exception { TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "file-preview", "-e", ePerson.getEmail(), "-p", PASSWORD }; + String[] args = new String[] { "file-preview", "-e", ePerson.getEmail() }; int run = ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), testDSpaceRunnableHandler, kernelImpl); assertEquals(0, run); @@ -129,7 +119,8 @@ public void testWhenNoFilesRun() throws Exception { public void testForSpecificItem() throws Exception { Item item2 = createOtherWorkspaceItemWithBitstream(ePerson, 0); // Run the script - runScriptForItemWithBitstreams(item2, ePerson, PASSWORD); + TestDSpaceRunnableHandler testHandler = runScriptForItemWithBitstreams(item2, ePerson); + checkHandlerMessages(testHandler, ePerson, item2, "logos.tgz", true); Bitstream b = bitstreamService.findAll(context).stream() .filter(bitstream -> bitstream.getName().equals("logos.tgz")) @@ -147,7 +138,8 @@ public void testForSpecificItem() throws Exception { public void testWhenScriptCannotCreateFilePreview() throws Exception { Item item2 = createOtherWorkspaceItemWithBitstream(eperson, 0); // Run the script as another user, without admin rights - runScriptForItemWithBitstreams(item2, ePerson, PASSWORD); + TestDSpaceRunnableHandler testHandler = runScriptForItemWithBitstreams(item2, ePerson); + checkHandlerMessages(testHandler, ePerson, item2, null, false); Bitstream b = bitstreamService.findAll(context).stream() .filter(bitstream -> bitstream.getName().equals("logos.tgz")) @@ -161,7 +153,8 @@ public void testWhenScriptCannotCreateFilePreview() throws Exception { assertFalse("Expects preview content not created.", previewContentService.hasPreview(context, b)); // Run the script as admin user - runScriptForItemWithBitstreams(item2, admin, password); + testHandler = runScriptForItemWithBitstreams(item2, admin); + checkHandlerMessages(testHandler, admin, item2, "logos.tgz", true); // now the preview content was created since the script was run by admin user assertTrue("Expects preview content created.", previewContentService.hasPreview(context, b)); @@ -173,7 +166,8 @@ public void testPreviewWithSyncStorage() throws Exception { configurationService.setProperty("sync.storage.service.enabled", true); Item item2 = createOtherWorkspaceItemWithBitstream(ePerson, SYNC_STORE_NUMBER); // Run the script - runScriptForItemWithBitstreams(item2, ePerson, PASSWORD); + TestDSpaceRunnableHandler testHandler = runScriptForItemWithBitstreams(item2, ePerson); + checkHandlerMessages(testHandler, ePerson, item2, "logos.tgz", true); Bitstream b = bitstreamService.findAll(context).stream() .filter(bitstream -> bitstream.getStoreNumber() == SYNC_STORE_NUMBER) @@ -188,7 +182,7 @@ public void testPreviewWithSyncStorage() throws Exception { public void testForAllItem() throws Exception { // Run the script TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); - String[] args = new String[] { "file-preview", "-e", ePerson.getEmail(), "-p", PASSWORD}; + String[] args = new String[] { "file-preview", "-e", ePerson.getEmail()}; int run = ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), testDSpaceRunnableHandler, kernelImpl); assertEquals(0, run); @@ -196,29 +190,68 @@ public void testForAllItem() throws Exception { checkNoError(testDSpaceRunnableHandler); } + @Test + public void testPreviewWithForce() throws Exception { + Item item2 = createOtherWorkspaceItemWithBitstream(ePerson, 0); + // Run the script + TestDSpaceRunnableHandler testHandler1 = runScriptForItemWithBitstreams(item2, ePerson); + checkHandlerMessages(testHandler1, ePerson, item2, "logos.tgz", true); + + // run again with force option, the existing preview content should be deleted and new one created + TestDSpaceRunnableHandler testHandler2 = new TestDSpaceRunnableHandler(); + String[] args = new String[] { "file-preview", "-u", item2.getID().toString(), + "-e", admin.getEmail(), "-f"}; + int run = ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), testHandler2, kernelImpl); + assertEquals(0, run); + checkNoError(testHandler2); + + List messages = testHandler2.getInfoMessages(); + assertThat(messages, hasSize(7)); + + assertThat(messages, hasItem(containsString("Deleting existing preview content:"))); + + Bitstream b = bitstreamService.findAll(context).stream() + .filter(bitstream -> bitstream.getName().equals("logos.tgz")) + .findFirst().orElse(null); + + assertTrue("Expects preview content created.", previewContentService.hasPreview(context, b)); + assertEquals(2, previewContentService.getPreview(context, b).size()); + } + private void checkNoError(TestDSpaceRunnableHandler testDSpaceRunnableHandler) { assertThat(testDSpaceRunnableHandler.getErrorMessages(), empty()); assertThat(testDSpaceRunnableHandler.getWarningMessages(), empty()); } - private void runScriptForItemWithBitstreams(Item item, EPerson user, String password) throws Exception { + private TestDSpaceRunnableHandler runScriptForItemWithBitstreams(Item item, EPerson user) + throws Exception { // Run the script TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); String[] args = new String[] { "file-preview", "-u", item.getID().toString(), - "-e", user.getEmail(), "-p", password}; + "-e", user.getEmail()}; int run = ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), testDSpaceRunnableHandler, kernelImpl); assertEquals(0, run); // There should be no errors or warnings checkNoError(testDSpaceRunnableHandler); - // There should be an info message about generating the file previews for the specified item + return testDSpaceRunnableHandler; + } + + private void checkHandlerMessages(TestDSpaceRunnableHandler testDSpaceRunnableHandler, + EPerson user, + Item item, + String fileName, + boolean previewGenerationExpected) { List messages = testDSpaceRunnableHandler.getInfoMessages(); - assertThat(messages, hasSize(2)); + assertThat(messages, hasSize(previewGenerationExpected ? 3 : 2)); assertThat(messages, hasItem(containsString("Generate the file previews for the specified item with " + "the given UUID: " + item.getID()))); - assertThat(messages, - hasItem(containsString("Authentication by user: " + user.getEmail()))); + assertThat(messages, hasItem(containsString("Authentication by user: " + user.getEmail()))); + if (previewGenerationExpected) { + // There should be an info message about generating the file previews for the specified bitstream + assertThat(messages, hasItem(containsString("Generating file preview for bitstream: " + fileName))); + } } private Item createOtherWorkspaceItemWithBitstream(EPerson user, int storageNumber) throws Exception { diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PreviewContentServiceImplIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PreviewContentServiceImplIT.java index 1249a029845a..47f7564b8eb3 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/PreviewContentServiceImplIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/PreviewContentServiceImplIT.java @@ -66,6 +66,7 @@ public class PreviewContentServiceImplIT extends AbstractControllerIntegrationTe Bitstream gzFile; Bitstream tarXzFile; Bitstream xzFile; + Bitstream tgzFileWithGzipMimeType; Bitstream tarGzFileWithWrongExtension; Bitstream tarXzFileWithIncorrectMimeType; @@ -133,6 +134,15 @@ public void setup() throws SQLException, AuthorizeException, IOException { .build(); } + try (InputStream is = getClass().getResourceAsStream("assetstore/logos.tgz")) { + tgzFileWithGzipMimeType = BitstreamBuilder. + createBitstream(context, bundle1, is) + .withName("logos.tgz") + .withDescription("tar.gz compressed file with tgz extension") + .withMimeType("application/x-gzip") + .build(); + } + try (InputStream is = getClass().getResourceAsStream("assetstore/logos.tgz")) { tgzFile = BitstreamBuilder. createBitstream(context, bundle1, is) @@ -235,18 +245,21 @@ public void destroy() throws Exception { BitstreamBuilder.deleteBitstream(tarGzFile.getID()); BitstreamFormat customMimeTypeFormat = tarXGzipFile.getFormat(context); - BitstreamBuilder.deleteBitstream(tarXGzipFile.getID()); - if (customMimeTypeFormat != null) { - bitstreamFormatService.delete(context, customMimeTypeFormat); - } + BitstreamBuilder.deleteBitstream(tarXGzipFile.getID()); BitstreamBuilder.deleteBitstream(tgzFile.getID()); BitstreamBuilder.deleteBitstream(gzFile.getID()); BitstreamBuilder.deleteBitstream(tarXzFile.getID()); BitstreamBuilder.deleteBitstream(xzFile.getID()); + BitstreamBuilder.deleteBitstream(tgzFileWithGzipMimeType.getID()); BitstreamBuilder.deleteBitstream(tarGzFileWithWrongExtension.getID()); BitstreamBuilder.deleteBitstream(tarXzFileWithIncorrectMimeType.getID()); + // removing custom mime type format created for tarXGzipFile and tgzFileWithGzipMimeType files + if (customMimeTypeFormat != null) { + bitstreamFormatService.delete(context, customMimeTypeFormat); + } + super.destroy(); } @@ -338,6 +351,11 @@ public void testXzContent() throws Exception { assertFileInfo(xzFile, "logos", 24); } + @Test + public void testTgzContentWithGzipMimetype() throws Exception { + assertFileInfos(tgzFileWithGzipMimeType); + } + @Test public void testGzContentForFileWithWrongExtension() throws Exception { assertFileInfo(tarGzFileWithWrongExtension, "TAR GZ File", 24); From eebe17dac8b91e1eb37a6f9efd9d1c77639a507c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Tue, 23 Jun 2026 11:11:13 +0200 Subject: [PATCH 33/41] [Port to dtq-dev] Issue 1343: add PUT and DELETE endpoint methods to ClarinLicenseLabel REST repository (#1325) * Issue 1343: add PUT and DELETE endpoint methods to ClarinLicenseLabel REST repository (ufal/clarin-dspace#1357) * Issue 1343: add PUT and DELETE endpoint methods to ClarinLicenseLabelRest repository * resolve Copilot comments * fixed PUT request in ClarinLicenseLabelRestRepository * added check for Clarin License Label -> Label string to be shorter that 5 characters * implement coorrect put method in ClarinLicenseLabelRestRepository * add constraints to license_label table: made label UNIQUE, make license_label not deletable when used in clarin licenses * change order of deleting objects in test cleanup(): delete license objects before license_label objects (to satisfy license_label constraints) * resolve PR Copilot comments, fixed failing ClarinWorkspaceItemRestRepositoryIT * prevent creating duplicate Clarin License Labels in REST API * not necessary to trim label twice * minor fixes, suggested by Copilot * Rename SQL migration files to use today's date (2026.06.01) --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> (cherry picked from commit b041c904b9b43770c668bc21086a407a6c5e3d1c) * Fix test compilation: update createClarinLicense call sites for new label arg The backport added a `label` String parameter to the createClarinLicense test helper but left six 4-arg call sites unchanged, breaking testCompile in dspace-server-webapp. Pass a label at each remaining call site ("lbl"; "lbl1"/"lbl2" for the paired-license test) to match the helper's new signature, preserving the previously hard-coded "lbl" behaviour. Co-Authored-By: Claude Opus 4.8 * PR comments: code cleanup --------- Co-authored-by: Milan Kuchtiak Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 --- .../clarin/ClarinLicenseLabelServiceImpl.java | 5 + .../clarin/ClarinLicenseServiceImpl.java | 5 + .../content/dao/clarin/ClarinLicenseDAO.java | 2 + .../dao/clarin/ClarinLicenseLabelDAO.java | 5 + .../dao/impl/clarin/ClarinLicenseDAOImpl.java | 20 ++ .../clarin/ClarinLicenseLabelDAOImpl.java | 17 ++ .../clarin/ClarinLicenseLabelService.java | 10 + .../service/clarin/ClarinLicenseService.java | 10 + ...6_2026.06.01__license_label_constraint.sql | 18 ++ ...6_2026.06.01__license_label_constraint.sql | 18 ++ .../util/AbstractBuilderCleanupUtil.java | 4 + .../ClarinLicenseLabelNotFoundException.java | 27 +++ .../ClarinLicenseLabelRestRepository.java | 108 ++++++++- .../ClarinLicenseLabelRestRepositoryIT.java | 222 +++++++++++++++++- .../ClarinWorkspaceItemRestRepositoryIT.java | 25 +- .../dspace/app/rest/ProvenanceServiceIT.java | 21 +- 16 files changed, 471 insertions(+), 46 deletions(-) create mode 100644 dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.6_2026.06.01__license_label_constraint.sql create mode 100644 dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/V7.6_2026.06.01__license_label_constraint.sql create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/ClarinLicenseLabelNotFoundException.java diff --git a/dspace-api/src/main/java/org/dspace/content/clarin/ClarinLicenseLabelServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/clarin/ClarinLicenseLabelServiceImpl.java index fce90020aafb..a56be61d6942 100644 --- a/dspace-api/src/main/java/org/dspace/content/clarin/ClarinLicenseLabelServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/clarin/ClarinLicenseLabelServiceImpl.java @@ -77,6 +77,11 @@ public List findAll(Context context) throws SQLException, Au return clarinLicenseLabelDAO.findAll(context, ClarinLicenseLabel.class); } + @Override + public ClarinLicenseLabel findByLabel(Context context, String label) throws SQLException { + return clarinLicenseLabelDAO.findByLabel(context, label); + } + @Override public void delete(Context context, ClarinLicenseLabel license) throws SQLException, AuthorizeException { if (!authorizeService.isAdmin(context)) { diff --git a/dspace-api/src/main/java/org/dspace/content/clarin/ClarinLicenseServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/clarin/ClarinLicenseServiceImpl.java index 10f209df7d04..e87f9365ccb8 100644 --- a/dspace-api/src/main/java/org/dspace/content/clarin/ClarinLicenseServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/clarin/ClarinLicenseServiceImpl.java @@ -98,6 +98,11 @@ public List findByNameLike(Context context, String name) throws S return clarinLicenseDAO.findByNameLike(context, name); } + @Override + public List findByLabel(Context context, String label) throws SQLException { + return clarinLicenseDAO.findByLabel(context, label); + } + @Override public void addLicenseMetadataToItem(Context context, ClarinLicense clarinLicense, Item item) throws SQLException { if (Objects.isNull(clarinLicense) || Objects.isNull(item)) { diff --git a/dspace-api/src/main/java/org/dspace/content/dao/clarin/ClarinLicenseDAO.java b/dspace-api/src/main/java/org/dspace/content/dao/clarin/ClarinLicenseDAO.java index 99147af64e65..9054947a9054 100644 --- a/dspace-api/src/main/java/org/dspace/content/dao/clarin/ClarinLicenseDAO.java +++ b/dspace-api/src/main/java/org/dspace/content/dao/clarin/ClarinLicenseDAO.java @@ -28,4 +28,6 @@ public interface ClarinLicenseDAO extends GenericDAO { List findByNameLike(Context context, String name) throws SQLException; + List findByLabel(Context context, String label) throws SQLException; + } diff --git a/dspace-api/src/main/java/org/dspace/content/dao/clarin/ClarinLicenseLabelDAO.java b/dspace-api/src/main/java/org/dspace/content/dao/clarin/ClarinLicenseLabelDAO.java index 1abd25b7a96a..d9444244e1c3 100644 --- a/dspace-api/src/main/java/org/dspace/content/dao/clarin/ClarinLicenseLabelDAO.java +++ b/dspace-api/src/main/java/org/dspace/content/dao/clarin/ClarinLicenseLabelDAO.java @@ -7,7 +7,10 @@ */ package org.dspace.content.dao.clarin; +import java.sql.SQLException; + import org.dspace.content.clarin.ClarinLicenseLabel; +import org.dspace.core.Context; import org.dspace.core.GenericDAO; /** @@ -19,4 +22,6 @@ * @author Milan Majchrak (milan.majchrak at dataquest.sk) */ public interface ClarinLicenseLabelDAO extends GenericDAO { + + ClarinLicenseLabel findByLabel(Context context, String label) throws SQLException; } diff --git a/dspace-api/src/main/java/org/dspace/content/dao/impl/clarin/ClarinLicenseDAOImpl.java b/dspace-api/src/main/java/org/dspace/content/dao/impl/clarin/ClarinLicenseDAOImpl.java index 24bbe180307c..93887a6918e2 100644 --- a/dspace-api/src/main/java/org/dspace/content/dao/impl/clarin/ClarinLicenseDAOImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/dao/impl/clarin/ClarinLicenseDAOImpl.java @@ -12,9 +12,13 @@ import javax.persistence.Query; import javax.persistence.criteria.CriteriaBuilder; import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Predicate; import javax.persistence.criteria.Root; +import javax.persistence.criteria.SetJoin; import org.dspace.content.clarin.ClarinLicense; +import org.dspace.content.clarin.ClarinLicenseLabel; +import org.dspace.content.clarin.ClarinLicenseLabel_; import org.dspace.content.clarin.ClarinLicense_; import org.dspace.content.dao.clarin.ClarinLicenseDAO; import org.dspace.core.AbstractHibernateDAO; @@ -54,4 +58,20 @@ public List findByNameLike(Context context, String name) throws S criteriaQuery.orderBy(criteriaBuilder.asc(clarinLicenseRoot.get(ClarinLicense_.name))); return list(context, criteriaQuery, false, ClarinLicense.class, -1, -1); } + + @Override + public List findByLabel(Context context, String label) throws SQLException { + CriteriaBuilder criteriaBuilder = getCriteriaBuilder(context); + CriteriaQuery criteriaQuery = getCriteriaQuery(criteriaBuilder, ClarinLicense.class); + Root clarinLicenseRoot = criteriaQuery.from(ClarinLicense.class); + + SetJoin labelJoin = + clarinLicenseRoot.joinSet(ClarinLicense_.CLARIN_LICENSE_LABELS); + + Predicate labelPredicate = criteriaBuilder.equal(labelJoin.get(ClarinLicenseLabel_.LABEL), label); + + criteriaQuery.select(clarinLicenseRoot).where(labelPredicate); + + return list(context, criteriaQuery, false, ClarinLicense.class, -1, -1); + } } diff --git a/dspace-api/src/main/java/org/dspace/content/dao/impl/clarin/ClarinLicenseLabelDAOImpl.java b/dspace-api/src/main/java/org/dspace/content/dao/impl/clarin/ClarinLicenseLabelDAOImpl.java index 1bf2179a3935..cb52af7c683d 100644 --- a/dspace-api/src/main/java/org/dspace/content/dao/impl/clarin/ClarinLicenseLabelDAOImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/dao/impl/clarin/ClarinLicenseLabelDAOImpl.java @@ -7,9 +7,16 @@ */ package org.dspace.content.dao.impl.clarin; +import java.sql.SQLException; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Root; + import org.dspace.content.clarin.ClarinLicenseLabel; +import org.dspace.content.clarin.ClarinLicenseLabel_; import org.dspace.content.dao.clarin.ClarinLicenseLabelDAO; import org.dspace.core.AbstractHibernateDAO; +import org.dspace.core.Context; /** * Hibernate implementation of the Database Access Object interface class for the Clarin License Label object. @@ -23,4 +30,14 @@ public class ClarinLicenseLabelDAOImpl extends AbstractHibernateDAO criteriaQuery = getCriteriaQuery(criteriaBuilder, ClarinLicenseLabel.class); + Root cllRoot = criteriaQuery.from(ClarinLicenseLabel.class); + criteriaQuery.select(cllRoot); + criteriaQuery.where(criteriaBuilder.equal(cllRoot.get(ClarinLicenseLabel_.label), label)); + return uniqueResult(context, criteriaQuery, true, ClarinLicenseLabel.class); + } } diff --git a/dspace-api/src/main/java/org/dspace/content/service/clarin/ClarinLicenseLabelService.java b/dspace-api/src/main/java/org/dspace/content/service/clarin/ClarinLicenseLabelService.java index adb56ecc238d..32e794622aad 100644 --- a/dspace-api/src/main/java/org/dspace/content/service/clarin/ClarinLicenseLabelService.java +++ b/dspace-api/src/main/java/org/dspace/content/service/clarin/ClarinLicenseLabelService.java @@ -52,6 +52,16 @@ ClarinLicenseLabel create(Context context, ClarinLicenseLabel clarinLicenseLabel */ ClarinLicenseLabel find(Context context, int valueId) throws SQLException; + /** + * Find the clarin license label object by label name + * + * @param context DSpace context object + * @param label label name of the searching clarin license label object + * @return found clarin license label object or null + * @throws SQLException if database error + */ + ClarinLicenseLabel findByLabel(Context context, String label) throws SQLException; + /** * Find all clarin license label objects * @param context DSpace context object diff --git a/dspace-api/src/main/java/org/dspace/content/service/clarin/ClarinLicenseService.java b/dspace-api/src/main/java/org/dspace/content/service/clarin/ClarinLicenseService.java index 93fbe88df3cb..1053d5741471 100644 --- a/dspace-api/src/main/java/org/dspace/content/service/clarin/ClarinLicenseService.java +++ b/dspace-api/src/main/java/org/dspace/content/service/clarin/ClarinLicenseService.java @@ -77,6 +77,16 @@ public interface ClarinLicenseService { */ List findByNameLike(Context context, String name) throws SQLException; + /** + * Find Clarin Licenses by the license label. + * + * @param context DSpace context object + * @param label the license label + * @return List of clarin licenses which contain the specified license label. + * @throws SQLException if database error + */ + List findByLabel(Context context, String label) throws SQLException; + void addLicenseMetadataToItem(Context context, ClarinLicense clarinLicense, Item item) throws SQLException; void clearLicenseMetadataFromItem(Context context, Item item) throws SQLException; diff --git a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.6_2026.06.01__license_label_constraint.sql b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.6_2026.06.01__license_label_constraint.sql new file mode 100644 index 000000000000..01bd6a493d3c --- /dev/null +++ b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.6_2026.06.01__license_label_constraint.sql @@ -0,0 +1,18 @@ +-- +-- The contents of this file are subject to the license and copyright +-- detailed in the LICENSE and NOTICE files at the root of the source +-- tree and available online at +-- +-- http://www.dspace.org/license/ +-- + +ALTER TABLE license_label_extended_mapping + DROP CONSTRAINT IF EXISTS license_label_license_label_extended_mapping_fk; + +-- here the "ON DELETE RESTRICT" clause (default clause) is used, which prevents deletion of a license_label record +-- when there are any license_label_extended_mapping records that reference it +ALTER TABLE license_label_extended_mapping + ADD CONSTRAINT license_label_license_label_extended_mapping_fk FOREIGN KEY (label_id) REFERENCES license_label(label_id); + +ALTER TABLE license_label DROP CONSTRAINT IF EXISTS license_label_label_unique; +ALTER TABLE license_label ADD CONSTRAINT license_label_label_unique UNIQUE(label); diff --git a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/V7.6_2026.06.01__license_label_constraint.sql b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/V7.6_2026.06.01__license_label_constraint.sql new file mode 100644 index 000000000000..01bd6a493d3c --- /dev/null +++ b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/V7.6_2026.06.01__license_label_constraint.sql @@ -0,0 +1,18 @@ +-- +-- The contents of this file are subject to the license and copyright +-- detailed in the LICENSE and NOTICE files at the root of the source +-- tree and available online at +-- +-- http://www.dspace.org/license/ +-- + +ALTER TABLE license_label_extended_mapping + DROP CONSTRAINT IF EXISTS license_label_license_label_extended_mapping_fk; + +-- here the "ON DELETE RESTRICT" clause (default clause) is used, which prevents deletion of a license_label record +-- when there are any license_label_extended_mapping records that reference it +ALTER TABLE license_label_extended_mapping + ADD CONSTRAINT license_label_license_label_extended_mapping_fk FOREIGN KEY (label_id) REFERENCES license_label(label_id); + +ALTER TABLE license_label DROP CONSTRAINT IF EXISTS license_label_label_unique; +ALTER TABLE license_label ADD CONSTRAINT license_label_label_unique UNIQUE(label); diff --git a/dspace-api/src/test/java/org/dspace/builder/util/AbstractBuilderCleanupUtil.java b/dspace-api/src/test/java/org/dspace/builder/util/AbstractBuilderCleanupUtil.java index 7ff2ff720017..18f1f82c290a 100644 --- a/dspace-api/src/test/java/org/dspace/builder/util/AbstractBuilderCleanupUtil.java +++ b/dspace-api/src/test/java/org/dspace/builder/util/AbstractBuilderCleanupUtil.java @@ -17,6 +17,8 @@ import org.dspace.builder.BitstreamFormatBuilder; import org.dspace.builder.BundleBuilder; import org.dspace.builder.ClaimedTaskBuilder; +import org.dspace.builder.ClarinLicenseBuilder; +import org.dspace.builder.ClarinLicenseLabelBuilder; import org.dspace.builder.CollectionBuilder; import org.dspace.builder.CommunityBuilder; import org.dspace.builder.EPersonBuilder; @@ -85,6 +87,8 @@ private void initMap() { map.put(SiteBuilder.class.getName(), new ArrayList<>()); map.put(ProcessBuilder.class.getName(), new ArrayList<>()); map.put(PreviewContentBuilder.class.getName(), new ArrayList<>()); + map.put(ClarinLicenseBuilder.class.getName(), new ArrayList<>()); + map.put(ClarinLicenseLabelBuilder.class.getName(), new ArrayList<>()); } /** diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/ClarinLicenseLabelNotFoundException.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/ClarinLicenseLabelNotFoundException.java new file mode 100644 index 000000000000..88da6a6f24aa --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/exception/ClarinLicenseLabelNotFoundException.java @@ -0,0 +1,27 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.exception; + +import javax.ws.rs.NotFoundException; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; + +/** + * Exception thrown when Clarin License Label not found + * + * @author Milan Kuchtiak + */ +@ResponseStatus(HttpStatus.NOT_FOUND) +public class ClarinLicenseLabelNotFoundException extends NotFoundException { + + public ClarinLicenseLabelNotFoundException(String message) { + super(message); + } + +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ClarinLicenseLabelRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ClarinLicenseLabelRestRepository.java index 061ef5807529..48190c6afc6c 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ClarinLicenseLabelRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/ClarinLicenseLabelRestRepository.java @@ -13,14 +13,19 @@ import java.sql.SQLException; import java.util.List; import java.util.Objects; +import java.util.Optional; +import javax.servlet.http.HttpServletRequest; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import org.dspace.app.rest.exception.ClarinLicenseLabelNotFoundException; import org.dspace.app.rest.exception.DSpaceBadRequestException; -import org.dspace.app.rest.exception.UnprocessableEntityException; import org.dspace.app.rest.model.ClarinLicenseLabelRest; import org.dspace.authorize.AuthorizeException; +import org.dspace.content.clarin.ClarinLicense; import org.dspace.content.clarin.ClarinLicenseLabel; import org.dspace.content.service.clarin.ClarinLicenseLabelService; +import org.dspace.content.service.clarin.ClarinLicenseService; import org.dspace.core.Context; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; @@ -36,9 +41,17 @@ @Component(ClarinLicenseLabelRest.CATEGORY + "." + ClarinLicenseLabelRest.NAME) public class ClarinLicenseLabelRestRepository extends DSpaceRestRepository { + private static final int MAX_LABEL_LENGTH = 5; + + @Autowired + ClarinLicenseService clarinLicenseService; + @Autowired ClarinLicenseLabelService clarinLicenseLabelService; + @Autowired + ObjectMapper objectMapper; + @Override public ClarinLicenseLabelRest findOne(Context context, Integer id) { ClarinLicenseLabel clarinLicenseLabel; @@ -72,7 +85,7 @@ protected ClarinLicenseLabelRest createAndReturn(Context context) // parse request body ClarinLicenseLabelRest clarinLicenseLabelRest; try { - clarinLicenseLabelRest = new ObjectMapper().readValue( + clarinLicenseLabelRest = objectMapper.readValue( getRequestService().getCurrentRequest().getHttpServletRequest().getInputStream(), ClarinLicenseLabelRest.class ); @@ -80,27 +93,98 @@ protected ClarinLicenseLabelRest createAndReturn(Context context) throw new DSpaceBadRequestException("error parsing request body", excIO); } - // validate fields - if (isBlank(clarinLicenseLabelRest.getLabel()) || isBlank(clarinLicenseLabelRest.getTitle())) { - throw new UnprocessableEntityException("CLARIN License Label title, label, icon cannot be null or empty"); + checkLabelAndTitle(clarinLicenseLabelRest); + if (clarinLicenseLabelService.findByLabel(context, clarinLicenseLabelRest.getLabel().trim()) != null) { + throw new DSpaceBadRequestException("Clarin License Label with label " + clarinLicenseLabelRest.getLabel() + + " already exists"); } // create ClarinLicenseLabel clarinLicenseLabel; clarinLicenseLabel = clarinLicenseLabelService.create(context); -// if (Objects.nonNull(clarinLicenseLabelRest.getId())) { -// clarinLicenseLabel.setId(clarinLicenseLabelRest.getId()); -// } - clarinLicenseLabel.setLabel(clarinLicenseLabelRest.getLabel()); - clarinLicenseLabel.setTitle(clarinLicenseLabelRest.getTitle()); - clarinLicenseLabel.setIcon(clarinLicenseLabelRest.getIcon()); - clarinLicenseLabel.setExtended(clarinLicenseLabelRest.isExtended()); + updateClarinLicenseLabel(clarinLicenseLabel, clarinLicenseLabelRest); clarinLicenseLabelService.update(context, clarinLicenseLabel); // return return converter.toRest(clarinLicenseLabel, utils.obtainProjection()); } + @Override + @PreAuthorize("hasAuthority('ADMIN')") + public ClarinLicenseLabelRest put(Context context, + HttpServletRequest request, + String apiCategory, + String model, + Integer id, + JsonNode jsonNode) throws SQLException, AuthorizeException { + ClarinLicenseLabel clarinLicenseLabel = clarinLicenseLabelService.find(context, id); + if (Objects.isNull(clarinLicenseLabel)) { + throw new ClarinLicenseLabelNotFoundException("Clarin License Label with id " + id + " was not found"); + } + + // parse request body + ClarinLicenseLabelRest clarinLicenseLabelRest; + try { + clarinLicenseLabelRest = objectMapper.treeToValue(jsonNode, ClarinLicenseLabelRest.class); + } catch (IOException excIO) { + throw new DSpaceBadRequestException("error parsing request body", excIO); + } + + checkLabelAndTitle(clarinLicenseLabelRest); + + ClarinLicenseLabel clarinLicenseLabelWithSameLabel = clarinLicenseLabelService.findByLabel(context, + clarinLicenseLabelRest.getLabel().trim()); + if (clarinLicenseLabelWithSameLabel != null && !clarinLicenseLabelWithSameLabel.getID().equals(id)) { + throw new DSpaceBadRequestException("Clarin License Label with label " + clarinLicenseLabelRest.getLabel() + + " already exists"); + } + + updateClarinLicenseLabel(clarinLicenseLabel, clarinLicenseLabelRest); + + clarinLicenseLabelService.update(context, clarinLicenseLabel); + + return converter.toRest(clarinLicenseLabel, utils.obtainProjection()); + } + + @Override + @PreAuthorize("hasAuthority('ADMIN')") + public void delete(Context context, Integer id) throws AuthorizeException { + ClarinLicenseLabel clarinLicenseLabel; + try { + clarinLicenseLabel = clarinLicenseLabelService.find(context, id); + if (Objects.isNull(clarinLicenseLabel)) { + throw new ClarinLicenseLabelNotFoundException("Clarin License Label with id " + id + " was not found"); + } + List licenses = clarinLicenseService.findByLabel(context, clarinLicenseLabel.getLabel()); + if (!licenses.isEmpty()) { + throw new DSpaceBadRequestException("Clarin License Label " + clarinLicenseLabel.getLabel() + + " is in use and cannot be deleted"); + } + clarinLicenseLabelService.delete(context, clarinLicenseLabel); + } catch (SQLException e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + private void checkLabelAndTitle(ClarinLicenseLabelRest clarinLicenseLabelRest) { + String label = Optional.ofNullable(clarinLicenseLabelRest.getLabel()).map(String::trim).orElse(null); + // validate fields + if (isBlank(label) || isBlank(clarinLicenseLabelRest.getTitle())) { + throw new DSpaceBadRequestException("CLARIN License Label title and label cannot be null or empty"); + } + if (label.length() > MAX_LABEL_LENGTH) { + throw new DSpaceBadRequestException( + "CLARIN License Label -> label string cannot be longer than " + MAX_LABEL_LENGTH + " characters"); + } + } + + private static void updateClarinLicenseLabel(ClarinLicenseLabel clarinLicenseLabel, + ClarinLicenseLabelRest clarinLicenseLabelRest) { + clarinLicenseLabel.setLabel(clarinLicenseLabelRest.getLabel().trim()); + clarinLicenseLabel.setTitle(clarinLicenseLabelRest.getTitle()); + clarinLicenseLabel.setIcon(clarinLicenseLabelRest.getIcon()); + clarinLicenseLabel.setExtended(clarinLicenseLabelRest.isExtended()); + } @Override public Class getDomainClass() { diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinLicenseLabelRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinLicenseLabelRestRepositoryIT.java index 207142d58e7d..2b59e2030263 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinLicenseLabelRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinLicenseLabelRestRepositoryIT.java @@ -10,13 +10,17 @@ import static com.jayway.jsonpath.JsonPath.read; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import java.util.Objects; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -25,9 +29,12 @@ import org.dspace.app.rest.model.ClarinLicenseLabelRest; import org.dspace.app.rest.projection.Projection; import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.builder.ClarinLicenseBuilder; import org.dspace.builder.ClarinLicenseLabelBuilder; +import org.dspace.content.clarin.ClarinLicense; import org.dspace.content.clarin.ClarinLicenseLabel; import org.dspace.content.service.clarin.ClarinLicenseLabelService; +import org.dspace.content.service.clarin.ClarinLicenseService; import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.Before; @@ -41,12 +48,18 @@ */ public class ClarinLicenseLabelRestRepositoryIT extends AbstractControllerIntegrationTest { + @Autowired + ClarinLicenseService clarinLicenseService; + @Autowired ClarinLicenseLabelService clarinLicenseLabelService; @Autowired ClarinLicenseLabelConverter clarinLicenseLabelConverter; + @Autowired + private ObjectMapper objectMapper; + ClarinLicenseLabel firstCLicenseLabel; ClarinLicenseLabel secondCLicenseLabel; ClarinLicenseLabel thirdCLicenseLabel; @@ -107,24 +120,18 @@ public void findAll() throws Exception { @Test public void create() throws Exception { - // create a new clarin license label - context.turnOffAuthorisationSystem(); - ClarinLicenseLabel clarinLicenseLabel = ClarinLicenseLabelBuilder.createClarinLicenseLabel(context).build(); - clarinLicenseLabel.setLabel("new"); - clarinLicenseLabel.setExtended(true); - clarinLicenseLabel.setTitle("New CLL"); - clarinLicenseLabel.setIcon(new byte[100]); - - ClarinLicenseLabelRest clarinLicenseLabelRest = clarinLicenseLabelConverter.convert(clarinLicenseLabel, - Projection.DEFAULT); - context.restoreAuthSystemState(); + ClarinLicenseLabelRest clarinLicenseLabelRest = new ClarinLicenseLabelRest(); + clarinLicenseLabelRest.setLabel("new"); + clarinLicenseLabelRest.setExtended(true); + clarinLicenseLabelRest.setTitle("New CLL"); + clarinLicenseLabelRest.setIcon(new byte[100]); // id of created clarin license AtomicReference idRef = new AtomicReference<>(); String authTokenAdmin = getAuthToken(admin.getEmail(), password); try { getClient(authTokenAdmin).perform(post("/api/core/clarinlicenselabels") - .content(new ObjectMapper().writeValueAsBytes(clarinLicenseLabelRest)) + .content(objectMapper.writeValueAsBytes(clarinLicenseLabelRest)) .contentType(org.springframework.http.MediaType.APPLICATION_JSON)) .andExpect(status().isCreated()) .andExpect(jsonPath("$.label", is(clarinLicenseLabelRest.getLabel()))) @@ -146,4 +153,195 @@ public void create() throws Exception { } } } + + @Test + public void createWithLongLabel() throws Exception { + ClarinLicenseLabelRest clarinLicenseLabelRest = new ClarinLicenseLabelRest(); + clarinLicenseLabelRest.setLabel("LONG_LABEL"); + clarinLicenseLabelRest.setExtended(true); + clarinLicenseLabelRest.setTitle("LONG CLL"); + clarinLicenseLabelRest.setIcon(new byte[100]); + + String authTokenAdmin = getAuthToken(admin.getEmail(), password); + getClient(authTokenAdmin).perform(post("/api/core/clarinlicenselabels") + .content(objectMapper.writeValueAsBytes(clarinLicenseLabelRest)) + .contentType(org.springframework.http.MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()); + } + + @Test + public void createWithDuplicateLabel() throws Exception { + ClarinLicenseLabelRest clarinLicenseLabelRest = new ClarinLicenseLabelRest(); + clarinLicenseLabelRest.setLabel(firstCLicenseLabel.getLabel()); + clarinLicenseLabelRest.setExtended(true); + clarinLicenseLabelRest.setTitle("Title 1"); + clarinLicenseLabelRest.setIcon(new byte[100]); + + String authTokenAdmin = getAuthToken(admin.getEmail(), password); + getClient(authTokenAdmin).perform(post("/api/core/clarinlicenselabels") + .content(objectMapper.writeValueAsBytes(clarinLicenseLabelRest)) + .contentType(org.springframework.http.MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()); + } + + @Test + public void updateOk() throws Exception { + String authTokenAdmin = getAuthToken(admin.getEmail(), password); + Integer clarinLicenseLabelId = null; + try { + context.turnOffAuthorisationSystem(); + ClarinLicenseLabel clarinLicenseLabel = ClarinLicenseLabelBuilder.createClarinLicenseLabel(context).build(); + clarinLicenseLabel.setLabel("CLL"); + clarinLicenseLabel.setExtended(true); + clarinLicenseLabel.setTitle("CLL Title4"); + clarinLicenseLabelService.update(context, clarinLicenseLabel); + + clarinLicenseLabelId = Objects.requireNonNull(clarinLicenseLabel.getID()); + context.restoreAuthSystemState(); + + ClarinLicenseLabelRest clarinLicenseLabelRest = clarinLicenseLabelConverter.convert(clarinLicenseLabel, + Projection.DEFAULT); + clarinLicenseLabelRest.setLabel("UPDATED CLL"); + clarinLicenseLabelRest.setTitle("Updated CLL Title"); + clarinLicenseLabelRest.setExtended(false); + + // test if the id from the path is used instead of the id from the body + clarinLicenseLabelRest.setId(999); + + // check if update ends with Bad Request since the label length is greater than 5 + getClient(authTokenAdmin).perform(put("/api/core/clarinlicenselabels/" + clarinLicenseLabelId) + .content(objectMapper.writeValueAsBytes(clarinLicenseLabelRest)) + .contentType(org.springframework.http.MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()); + + // set the label to valid value and check if the update is successful + clarinLicenseLabelRest.setLabel(" CLL-X "); + getClient(authTokenAdmin).perform(put("/api/core/clarinlicenselabels/" + clarinLicenseLabelId) + .content(objectMapper.writeValueAsBytes(clarinLicenseLabelRest)) + .contentType(org.springframework.http.MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.id", is(clarinLicenseLabelId))) + .andExpect(jsonPath("$.label", is("CLL-X"))) + .andExpect(jsonPath("$.title", is("Updated CLL Title"))) + .andExpect(jsonPath("$.extended", is(false))) + .andExpect(jsonPath("$.icon", nullValue())) + .andExpect(jsonPath("$.type", is(ClarinLicenseLabelRest.NAME))); + + getClient(authTokenAdmin).perform(get("/api/core/clarinlicenselabels/" + clarinLicenseLabelId)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.id", is(clarinLicenseLabelId))) + .andExpect(jsonPath("$.label", is("CLL-X"))) + .andExpect(jsonPath("$.title", is("Updated CLL Title"))) + .andExpect(jsonPath("$.extended", is(false))) + .andExpect(jsonPath("$.icon", nullValue())) + .andExpect(jsonPath("$.type", is(ClarinLicenseLabelRest.NAME))); + } finally { + ClarinLicenseLabelBuilder.deleteClarinLicenseLabel(clarinLicenseLabelId); + } + } + + @Test + public void updateNotFound() throws Exception { + String authTokenAdmin = getAuthToken(admin.getEmail(), password); + getClient(authTokenAdmin).perform(put("/api/core/clarinlicenselabels/999") + .content("{}") + .contentType(org.springframework.http.MediaType.APPLICATION_JSON)) + .andExpect(status().isNotFound()); + } + + @Test + public void updateInvalidBody() throws Exception { + String authTokenAdmin = getAuthToken(admin.getEmail(), password); + getClient(authTokenAdmin).perform(put("/api/core/clarinlicenselabels/" + firstCLicenseLabel.getID()) + .content("{\"label\": \"lbl\", \"invalid_property\": 0}") + .contentType(org.springframework.http.MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()); + } + + @Test + public void updateMissingTitle() throws Exception { + String authTokenAdmin = getAuthToken(admin.getEmail(), password); + getClient(authTokenAdmin).perform(put("/api/core/clarinlicenselabels/" + firstCLicenseLabel.getID()) + .content("{\"label\": \"test label\"}") + .contentType(org.springframework.http.MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()); + } + + @Test + public void updateNotAuthorized() throws Exception { + getClient().perform(put("/api/core/clarinlicenselabels/999") + .content("{}") + .contentType(org.springframework.http.MediaType.APPLICATION_JSON)) + .andExpect(status().isUnauthorized()); + } + + @Test + public void updateWithDuplicateLabel() throws Exception { + ClarinLicenseLabelRest clarinLicenseLabelRest = new ClarinLicenseLabelRest(); + clarinLicenseLabelRest.setLabel(firstCLicenseLabel.getLabel()); + clarinLicenseLabelRest.setExtended(true); + clarinLicenseLabelRest.setTitle("Title 1"); + clarinLicenseLabelRest.setIcon(new byte[100]); + + String authTokenAdmin = getAuthToken(admin.getEmail(), password); + + getClient(authTokenAdmin).perform(put("/api/core/clarinlicenselabels/" + secondCLicenseLabel.getID()) + .content(objectMapper.writeValueAsBytes(clarinLicenseLabelRest)) + .contentType(org.springframework.http.MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()); + + // set the label to the same value as the secondCLicenseLabel and check if the update is successful + clarinLicenseLabelRest.setLabel(secondCLicenseLabel.getLabel()); + getClient(authTokenAdmin).perform(put("/api/core/clarinlicenselabels/" + secondCLicenseLabel.getID()) + .content(objectMapper.writeValueAsBytes(clarinLicenseLabelRest)) + .contentType(org.springframework.http.MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + } + + @Test + public void deleteNotAuthorized() throws Exception { + getClient().perform(delete("/api/core/clarinlicenselabels/" + firstCLicenseLabel.getID())) + .andExpect(status().isUnauthorized()); + } + + @Test + public void deleteOk() throws Exception { + String authTokenAdmin = getAuthToken(admin.getEmail(), password); + getClient(authTokenAdmin).perform(delete("/api/core/clarinlicenselabels/" + firstCLicenseLabel.getID())) + .andExpect(status().isNoContent()); + } + + @Test + public void deleteNotFound() throws Exception { + String authTokenAdmin = getAuthToken(admin.getEmail(), password); + getClient(authTokenAdmin).perform(delete("/api/core/clarinlicenselabels/999")) + .andExpect(status().isNotFound()); + } + + @Test + public void deleteForLabelUsed() throws Exception { + // create ClarinLicense + context.turnOffAuthorisationSystem(); + ClarinLicense firstCLicense = ClarinLicenseBuilder.createClarinLicense(context).build(); + firstCLicense.setName("CL Name1"); + firstCLicense.setConfirmation(ClarinLicense.Confirmation.NOT_REQUIRED); + firstCLicense.setDefinition("CL Definition1"); + firstCLicense.setRequiredInfo("CL Req1"); + // add ClarinLicenseLabels to the ClarinLicense + firstCLicense.setLicenseLabels(Set.of(firstCLicenseLabel, thirdCLicenseLabel)); + clarinLicenseService.update(context, firstCLicense); + context.restoreAuthSystemState(); + + String authTokenAdmin = getAuthToken(admin.getEmail(), password); + + getClient(authTokenAdmin).perform(delete("/api/core/clarinlicenselabels/" + firstCLicenseLabel.getID())) + .andExpect(status().isBadRequest()); + getClient(authTokenAdmin).perform(delete("/api/core/clarinlicenselabels/" + thirdCLicenseLabel.getID())) + .andExpect(status().isBadRequest()); + + context.turnOffAuthorisationSystem(); + clarinLicenseService.delete(context, firstCLicense); + context.restoreAuthSystemState(); + } + } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinWorkspaceItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinWorkspaceItemRestRepositoryIT.java index aae481283540..b5ab117c4e80 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinWorkspaceItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinWorkspaceItemRestRepositoryIT.java @@ -638,7 +638,7 @@ public void addClarinLicenseToWI() throws Exception { String clarinLicenseName = "Test Clarin License"; // 2. Create clarin license with clarin license label - ClarinLicense clarinLicense = createClarinLicense(clarinLicenseName, "Test Def", "Test R Info", + ClarinLicense clarinLicense = createClarinLicense(clarinLicenseName, "Test Def", "Test R Info", "lbl", Confirmation.NOT_REQUIRED); // creating replace operation @@ -688,7 +688,7 @@ public void removeClarinLicenseFromWI() throws Exception { List replaceOperations = new ArrayList(); // 2. Create Clarin License String clarinLicenseName = "Test Clarin License"; - ClarinLicense clarinLicense = createClarinLicense(clarinLicenseName, "Test Def", "Test R Info", + ClarinLicense clarinLicense = createClarinLicense(clarinLicenseName, "Test Def", "Test R Info", "lbl", Confirmation.NOT_REQUIRED); context.restoreAuthSystemState(); @@ -763,10 +763,11 @@ public void updateClarinLicenseInWI() throws Exception { String updateClarinLicenseName = "Updated Clarin License"; // 2. Create Clarin Licenses - ClarinLicense clarinLicense = createClarinLicense(clarinLicenseName, "Test Def", "Test R Info", + ClarinLicense clarinLicense = createClarinLicense(clarinLicenseName, "Test Def", "Test R Info", "lbl1", Confirmation.NOT_REQUIRED); ClarinLicense updatedClarinLicense = - createClarinLicense(updateClarinLicenseName, "Test Def2", "Test R Info2", Confirmation.NOT_REQUIRED); + createClarinLicense(updateClarinLicenseName, "Test Def2", "Test R Info2", "lbl2", + Confirmation.NOT_REQUIRED); context.restoreAuthSystemState(); // Creating replace operation @@ -832,7 +833,7 @@ public void addClarinLicenseViaSectionPatch() throws Exception { String clarinLicenseName = "Test Section Clarin License"; ClarinLicense clarinLicense = createClarinLicense(clarinLicenseName, "Test Def", "Test R Info", - Confirmation.NOT_REQUIRED); + "lbl", Confirmation.NOT_REQUIRED); context.restoreAuthSystemState(); List replaceOperations = new ArrayList(); @@ -870,7 +871,7 @@ public void getWorkspaceItemReturnsDistinctLicenseSections() throws Exception { String clarinLicenseName = "Distinct Sections Clarin License"; createClarinLicense(clarinLicenseName, "Test Def", "Test R Info", - Confirmation.NOT_REQUIRED); + "lbl", Confirmation.NOT_REQUIRED); context.restoreAuthSystemState(); // Apply the CLARIN license through the section-scoped path @@ -912,7 +913,7 @@ public void patchSelectWithEmptyValueClearsLicense() throws Exception { String clarinLicenseName = "Empty Value Clarin License"; ClarinLicense clarinLicense = createClarinLicense(clarinLicenseName, "Test Def", "Test R Info", - Confirmation.NOT_REQUIRED); + "lbl", Confirmation.NOT_REQUIRED); context.restoreAuthSystemState(); String tokenAdmin = getAuthToken(admin.getEmail(), password); @@ -958,8 +959,8 @@ public void patchSelectReplacesPreviousLicense() throws Exception { String firstName = "First Clarin License"; String secondName = "Second Clarin License"; - ClarinLicense first = createClarinLicense(firstName, "Def1", "Info1", Confirmation.NOT_REQUIRED); - ClarinLicense second = createClarinLicense(secondName, "Def2", "Info2", Confirmation.NOT_REQUIRED); + ClarinLicense first = createClarinLicense(firstName, "Def1", "Info1", "lbl1", Confirmation.NOT_REQUIRED); + ClarinLicense second = createClarinLicense(secondName, "Def2", "Info2", "lbl2", Confirmation.NOT_REQUIRED); context.restoreAuthSystemState(); String tokenAdmin = getAuthToken(admin.getEmail(), password); @@ -1041,7 +1042,7 @@ public void patchSelectAsAnonymousIsUnauthorized() throws Exception { context.turnOffAuthorisationSystem(); WorkspaceItem witem = createWorkspaceItemWithFile(); String clarinLicenseName = "Anon Clarin License"; - createClarinLicense(clarinLicenseName, "Def", "Info", Confirmation.NOT_REQUIRED); + createClarinLicense(clarinLicenseName, "Def", "Info", "lbl", Confirmation.NOT_REQUIRED); context.restoreAuthSystemState(); List ops = new ArrayList(); @@ -1257,7 +1258,7 @@ private ClarinLicenseLabel createClarinLicenseLabel(String label, boolean extend /** * Create ClarinLicense object with ClarinLicenseLabel object for testing purposes. */ - private ClarinLicense createClarinLicense(String name, String definition, String requiredInfo, + private ClarinLicense createClarinLicense(String name, String definition, String requiredInfo, String label, Confirmation confirmation) throws SQLException, AuthorizeException { ClarinLicense clarinLicense = ClarinLicenseBuilder.createClarinLicense(context).build(); clarinLicense.setConfirmation(confirmation); @@ -1267,7 +1268,7 @@ private ClarinLicense createClarinLicense(String name, String definition, String // add ClarinLicenseLabels to the ClarinLicense HashSet clarinLicenseLabels = new HashSet<>(); - ClarinLicenseLabel clarinLicenseLabel = createClarinLicenseLabel("lbl", false, "Test Title"); + ClarinLicenseLabel clarinLicenseLabel = createClarinLicenseLabel(label, false, label + " Title"); clarinLicenseLabels.add(clarinLicenseLabel); clarinLicense.setLicenseLabels(clarinLicenseLabels); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ProvenanceServiceIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ProvenanceServiceIT.java index da7cd789ccca..1f0c83d55462 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ProvenanceServiceIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ProvenanceServiceIT.java @@ -121,9 +121,9 @@ public void destroy() throws Exception { @Test public void updateLicenseTest() throws Exception { Bitstream bitstream = createBitstream(item, Constants.LICENSE_BUNDLE_NAME); - ClarinLicense clarinLicense1 = createClarinLicense("Test 1", "Test Def"); + ClarinLicense clarinLicense1 = createClarinLicense("Test 1", "Test Def", "LBL_1"); ClarinLicenseResourceMapping mapping = createResourceMapping(clarinLicense1, bitstream); - ClarinLicense clarinLicense2 = createClarinLicense("Test 2", "Test Def"); + ClarinLicense clarinLicense2 = createClarinLicense("Test 2", "Test Def", "LBL_2"); String token = getAuthToken(admin.getEmail(), password); getClient(token).perform(put("/api/core/items/" + item.getID() + "/bundles") @@ -139,7 +139,7 @@ public void updateLicenseTest() throws Exception { @Test public void addLicenseTest() throws Exception { - ClarinLicense clarinLicense = createClarinLicense("Test", "Test Def"); + ClarinLicense clarinLicense = createClarinLicense("Test", "Test Def", "LBL"); String token = getAuthToken(admin.getEmail(), password); getClient(token).perform(put("/api/core/items/" + item.getID() + "/bundles") @@ -153,7 +153,7 @@ public void addLicenseTest() throws Exception { @Test public void removeLicenseTest() throws Exception { Bitstream bitstream = createBitstream(item, Constants.LICENSE_BUNDLE_NAME); - ClarinLicense clarinLicense = createClarinLicense("Test", "Test Def"); + ClarinLicense clarinLicense = createClarinLicense("Test", "Test Def", "LBL"); ClarinLicenseResourceMapping mapping = createResourceMapping(clarinLicense, bitstream); String token = getAuthToken(admin.getEmail(), password); @@ -478,14 +478,14 @@ private ClarinLicenseLabel createClarinLicenseLabel(String label, boolean extend return clarinLicenseLabel; } - private ClarinLicense createClarinLicense(String name, String definition) + private ClarinLicense createClarinLicense(String name, String definition, String label) throws SQLException, AuthorizeException { context.turnOffAuthorisationSystem(); ClarinLicense clarinLicense = ClarinLicenseBuilder.createClarinLicense(context).build(); clarinLicense.setDefinition(definition); clarinLicense.setName(name); HashSet clarinLicenseLabels = new HashSet<>(); - ClarinLicenseLabel clarinLicenseLabel = createClarinLicenseLabel("lbl", false, "Test Title"); + ClarinLicenseLabel clarinLicenseLabel = createClarinLicenseLabel(label, false, label + " Title"); clarinLicenseLabels.add(clarinLicenseLabel); clarinLicense.setLicenseLabels(clarinLicenseLabels); clarinLicenseService.update(context, clarinLicense); @@ -498,11 +498,12 @@ private void deleteClarinLicenseLable(Integer id) throws Exception { } private void deleteClarinLicense(ClarinLicense license) throws Exception { - int size = license.getLicenseLabels().size(); - for (int i = 0; i < size; i++) { - deleteClarinLicenseLable(license.getLicenseLabels().get(i).getID()); - } + // first delete license, then labels, because of the foreign key constraint + List clarinLicenseLabels = license.getLicenseLabels(); ClarinLicenseBuilder.deleteClarinLicense(license.getID()); + for (ClarinLicenseLabel clarinLicenseLabel : clarinLicenseLabels) { + deleteClarinLicenseLable(clarinLicenseLabel.getID()); + } } private Collection createCollection() { From ee37116b26ebffbf3ad71eed44f9f12cb4ac10c6 Mon Sep 17 00:00:00 2001 From: jurinecko Date: Thu, 25 Jun 2026 10:16:27 +0200 Subject: [PATCH 34/41] AI-Skills/Wire private AI skills submodule (.dspace-skills) (#1343) --- .dspace-skills | 1 + .gitmodules | 4 ++++ AGENTS.md | 16 ++++++++++++++++ CLAUDE.md | 16 ++++++++++++++++ 4 files changed, 37 insertions(+) create mode 160000 .dspace-skills create mode 100644 .gitmodules create mode 100644 AGENTS.md create mode 100644 CLAUDE.md diff --git a/.dspace-skills b/.dspace-skills new file mode 160000 index 000000000000..2a62a2ebff88 --- /dev/null +++ b/.dspace-skills @@ -0,0 +1 @@ +Subproject commit 2a62a2ebff88a05acf78d374003032b455843066 diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000000..1050e4f30341 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule ".dspace-skills"] + path = .dspace-skills + url = git@github.com:dataquest-dev/dspace-skills.git + branch = main diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000000..5e55e477c0b6 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,16 @@ + + +## Private team playbooks (dataquest) + +This repo vendors dataquest's private AI knowledge base as the `.dspace-skills/` git submodule. **If +`.dspace-skills/` is present**, treat **`.dspace-skills/AGENTS.md`** as the authoritative agent guide for this repo: +read it first, then load the matching profile (`.dspace-skills/profiles/frontend.md` for dspace-angular, +`.dspace-skills/profiles/backend.md` for DSpace) and pull skills from `.dspace-skills/skills/` on demand. Start any +PR/backport/test task from `.dspace-skills/SKILLS.md`. + +If `.dspace-skills/` is empty (you don't have access, e.g. an outside contributor), ignore this section +and proceed with the public project conventions. + +To enable: `git submodule update --init .dspace-skills` (requires access to +`dataquest-dev/dspace-skills`). + diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000000..5e55e477c0b6 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,16 @@ + + +## Private team playbooks (dataquest) + +This repo vendors dataquest's private AI knowledge base as the `.dspace-skills/` git submodule. **If +`.dspace-skills/` is present**, treat **`.dspace-skills/AGENTS.md`** as the authoritative agent guide for this repo: +read it first, then load the matching profile (`.dspace-skills/profiles/frontend.md` for dspace-angular, +`.dspace-skills/profiles/backend.md` for DSpace) and pull skills from `.dspace-skills/skills/` on demand. Start any +PR/backport/test task from `.dspace-skills/SKILLS.md`. + +If `.dspace-skills/` is empty (you don't have access, e.g. an outside contributor), ignore this section +and proceed with the public project conventions. + +To enable: `git submodule update --init .dspace-skills` (requires access to +`dataquest-dev/dspace-skills`). + From f6f13561cb21fdcec78478b5824a7f4931ff56a9 Mon Sep 17 00:00:00 2001 From: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:17:44 +0200 Subject: [PATCH 35/41] test: de-flake ORCID cache tests and ZIP-download IT (#1344) * test: de-flake ORCID cache tests and ZIP-download IT Two independent flaky tests keep turning the dtq-dev pipeline red after #1321: 1. CachingOrcidRestConnectorTest.testCachable / testCacheableWithError still hit the live ORCID sandbox (#1321 only mocked getLabel/search/search_fail). They use the real Spring @Cacheable bean (to exercise the CGLIB caching proxy), so they could not be spied. Point the bean's apiURL at a local MockWebServer serving the canned orcid-expanded-search.xml instead -- keeps the real caching proxy and HTTP transport under test, removes the network dependency. No production change. 2. MetadataBitstreamControllerIT.downloadAllZip compared the response to a locally-built ZIP byte-for-byte; a ZIP entry's DOS timestamp (2s resolution) defaults to "now" on both sides and differs across a 2s boundary. Assert the unzipped entry name + content instead of raw bytes. Test-only changes. Verified locally (ORCID class 8/8 green, repeated offline runs; webapp test module compiles; checkstyle clean). Co-Authored-By: Claude Opus 4.8 * test: assert exactly one ZIP entry in downloadAllZip Address CodeRabbit: a Map keyed by entry name could mask duplicate ZIP entries (same filename overwrites). Track the entry count separately and assert it is 1, so an unexpected extra entry fails the test. Co-Authored-By: Claude Opus 4.8 * test: de-flake SSR authorization and Solr TopCountries ITs Two more intermittent failures on dtq-dev, both addressed at the root cause (test-only changes): 1. AuthorizationRestRepositoryIT.findByObjectSSRTest flaked with 400 instead of 200. The test sets dspace.server.ssr.url (used by Utils.getBaseObjectRestFromUri to resolve the request URI) and the AlwaysThrowExceptionFeature.turnoff flag via configurationService.setProperty(...). Such in-memory overrides are silently dropped when the combined config is rebuilt by the auto-reload listener (fires on any reloadable cfg file mtime change mid-run). When ssr.url is dropped the URI no longer resolves -> 400; if turnoff were also dropped the /search/object path would let alwaysexception throw -> 500. Fix: set both via JVM system properties (+ reloadConfig) so they sit in the highest-precedence override layer and survive auto-reload, cleared in @After. Same pattern as #1321's Shibboleth fix (AuthenticationRestControllerIT#setAuthenticationMethodSequence). Applied to all three SSR tests. 2. StatisticsRestRepositoryIT.topCountriesReport_Community_Visited flaked with an empty report (points: []). postView() commits with waitSearcher=false, so the just-posted view events can be invisible to the immediately-following report query (and a dropped solr-statistics.autoCommit override would skip the commit entirely). Fix: after posting the view events, force solrLoggerService.commit() (waitSearcher =true) so the events are flushed and visible before the report is queried. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- .../CachingOrcidRestConnectorTest.java | 87 ++++++++++++------- .../rest/AuthorizationRestRepositoryIT.java | 55 ++++++++++-- .../rest/MetadataBitstreamControllerIT.java | 55 +++++++----- .../app/rest/StatisticsRestRepositoryIT.java | 7 ++ 4 files changed, 142 insertions(+), 62 deletions(-) diff --git a/dspace-api/src/test/java/org/dspace/external/CachingOrcidRestConnectorTest.java b/dspace-api/src/test/java/org/dspace/external/CachingOrcidRestConnectorTest.java index 7e8cbc6c94fc..47d8af106079 100644 --- a/dspace-api/src/test/java/org/dspace/external/CachingOrcidRestConnectorTest.java +++ b/dspace-api/src/test/java/org/dspace/external/CachingOrcidRestConnectorTest.java @@ -19,7 +19,10 @@ import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; import org.dspace.AbstractDSpaceTest; import org.dspace.external.provider.orcid.xml.ExpandedSearchConverter; import org.dspace.utils.DSpace; @@ -53,6 +56,19 @@ private InputStream cannedResponse(String resource) { return is; } + /** + * Build a canned 200 OK ORCID "expanded-search" response for the mock HTTP server, so the cache-aware + * tests below exercise the real Spring {@code @Cacheable} bean without depending on the live ORCID sandbox. + */ + private MockResponse cannedOrcidResponse() throws IOException { + try (InputStream is = cannedResponse(EXPANDED_SEARCH_XML)) { + return new MockResponse() + .setResponseCode(200) + .setHeader("Content-Type", "application/vnd.orcid+xml") + .setBody(new String(is.readAllBytes(), StandardCharsets.UTF_8)); + } + } + @Before public void setup() { sut = new CachingOrcidRestConnector(); @@ -134,7 +150,7 @@ public void search_fail() throws Exception { } @Test - public void testCachable() { + public void testCachable() throws IOException { CachingOrcidRestConnector c = new DSpace().getServiceManager().getServiceByName( "CachingOrcidRestConnector", CachingOrcidRestConnector.class); @@ -148,43 +164,56 @@ public void testCachable() { verify(c, times(1)).getLabel(orcid); */ - c.setApiURL("https://pub.sandbox.orcid.org/v3.0"); - c.forceAccessToken(sandboxToken); - - String r1 = c.getLabel(orcid); - assertEquals(expectedLabel, r1); - String r2 = c.getLabel(orcid); - assertEquals(expectedLabel, r2); - //get the orcid-labels cache and verify that the label is there - assertEquals(expectedLabel, cache.get(orcid).get()); + // Drive the real Spring @Cacheable bean against a local mock HTTP server instead of the live ORCID + // sandbox, whose dataset is periodically reset and previously caused intermittent CI failures. + try (MockWebServer server = new MockWebServer()) { + // Two responses are enqueued, but with caching working only the FIRST getLabel() hits the server; + // the second is served from the "orcid-labels" cache (asserted via getRequestCount() below). + server.enqueue(cannedOrcidResponse()); + server.enqueue(cannedOrcidResponse()); + + c.setApiURL(server.url("/v3.0").toString()); + c.forceAccessToken(sandboxToken); + + String r1 = c.getLabel(orcid); + assertEquals(expectedLabel, r1); + String r2 = c.getLabel(orcid); + assertEquals(expectedLabel, r2); + //get the orcid-labels cache and verify that the label is there + assertEquals(expectedLabel, cache.get(orcid).get()); + //caching means two getLabel() calls produced a single ORCID API request + assertEquals("Expected getLabel to be cached after the first call", 1, server.getRequestCount()); + } } @Test - public void testCacheableWithError() { + public void testCacheableWithError() throws IOException { CachingOrcidRestConnector c = new DSpace().getServiceManager().getServiceByName( "CachingOrcidRestConnector", CachingOrcidRestConnector.class); Cache cache = prepareCache(); assertNull(cache.get(orcid)); - //skip init - c.forceAccessToken(sandboxToken); - //set bad ApiURL to provoke an error - c.setApiURL("https://api.sandbox.orcid.org/"); - String r1 = c.getLabel(orcid); - //on error, getLabel should return null - assertNull(r1); - //the cache should not contain a value for this id - assertNull(cache.get(orcid)); - - //fix the error - c.setApiURL("https://pub.sandbox.orcid.org/v3.0"); - // the error flipped the initialized flag, this reset it - c.forceAccessToken(sandboxToken); - String r2 = c.getLabel(orcid); - assertEquals(expectedLabel, r2); - //the cache should now contain a value for this id - assertEquals(expectedLabel, cache.get(orcid).get()); + try (MockWebServer server = new MockWebServer()) { + //the (mock) ORCID API returns an error first, then a valid response + server.enqueue(new MockResponse().setResponseCode(500)); + server.enqueue(cannedOrcidResponse()); + + //skip init (force a token so getAccessToken/init never reaches out to the network) + c.forceAccessToken(sandboxToken); + c.setApiURL(server.url("/v3.0").toString()); + String r1 = c.getLabel(orcid); + //on error, getLabel should return null + assertNull(r1); + //a null result must NOT be cached (see @Cacheable(unless = "#result == null")) + assertNull(cache.get(orcid)); + + //the second call gets the valid (200) response; the error never cleared the token, so no re-init needed + String r2 = c.getLabel(orcid); + assertEquals(expectedLabel, r2); + //the cache should now contain a value for this id + assertEquals(expectedLabel, cache.get(orcid).get()); + } } private Cache prepareCache() { diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/AuthorizationRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/AuthorizationRestRepositoryIT.java index 365e88597d8d..28bbf74a00dd 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/AuthorizationRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/AuthorizationRestRepositoryIT.java @@ -63,6 +63,7 @@ import org.dspace.eperson.Group; import org.dspace.services.ConfigurationService; import org.hamcrest.Matchers; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -134,6 +135,42 @@ public class AuthorizationRestRepositoryIT extends AbstractControllerIntegration */ private AuthorizationFeature trueForUsersInGroupTest; + private static final String SSR_URL_KEY = "dspace.server.ssr.url"; + private static final String SSR_URL = "http://ssr.example.com/api"; + private static final String ALWAYS_THROW_TURNOFF_KEY = + "org.dspace.app.rest.authorization.AlwaysThrowExceptionFeature.turnoff"; + + /** + * Configure the SSR object-by-URI resolution used by the {@code search/object} tests (disarm the + * AlwaysThrowExceptionFeature and define the SSR base URL), setting both as JVM system properties + * (+ {@link ConfigurationService#reloadConfig()}) instead of {@code configurationService.setProperty(...)}. + * + *

A plain {@code setProperty(...)} override only lives in the in-memory combined-config view and is + * silently dropped whenever that view is rebuilt by the auto-reload listener (which fires when any + * reloadable cfg file's mtime changes, e.g. another test writing {@code local.cfg}). When that rebuild + * lands mid-test the SSR url disappears and {@code search/object} no longer resolves the uri -> 400 (and a + * dropped turnoff would let {@code alwaysexception} throw -> 500). A system property sits in the + * highest-precedence override layer and is re-read on every rebuild, so it survives auto-reload; it is + * cleared in {@link #clearSsrObjectResolution()}. Same pattern as + * AuthenticationRestControllerIT#setAuthenticationMethodSequence.

+ */ + private void enableSsrObjectResolution() { + System.setProperty(ALWAYS_THROW_TURNOFF_KEY, "true"); + System.setProperty(SSR_URL_KEY, SSR_URL); + configurationService.reloadConfig(); + } + + /** + * Remove the system-property overrides set by {@link #enableSsrObjectResolution()} so they do not leak into + * other tests in the same JVM. Runs before the superclass {@code @After}, whose {@code reloadConfig()} then + * restores the on-disk defaults. + */ + @After + public void clearSsrObjectResolution() { + System.clearProperty(SSR_URL_KEY); + System.clearProperty(ALWAYS_THROW_TURNOFF_KEY); + } + @Override @Before public void setUp() throws Exception { @@ -852,9 +889,9 @@ public void findByObjectSSRTest() throws Exception { SiteRest siteRest = siteConverter.convert(site, DefaultProjection.DEFAULT); String siteUri = "http://ssr.example.com/api/core/sites/" + siteRest.getId(); - // disarm the alwaysThrowExceptionFeature - configurationService.setProperty("org.dspace.app.rest.authorization.AlwaysThrowExceptionFeature.turnoff", true); - configurationService.setProperty("dspace.server.ssr.url", "http://ssr.example.com/api"); + // Disarm the alwaysThrowExceptionFeature and define the SSR base URL via system properties so the + // overrides survive a mid-test config auto-reload (see enableSsrObjectResolution). + enableSsrObjectResolution(); String adminToken = getAuthToken(admin.getEmail(), password); String epersonToken = getAuthToken(eperson.getEmail(), password); @@ -969,9 +1006,9 @@ public void findByObjectSSRTest() throws Exception { */ @Test public void findByObjectBadRequestSSRTest() throws Exception { - // disarm the alwaysThrowExceptionFeature - configurationService.setProperty("org.dspace.app.rest.authorization.AlwaysThrowExceptionFeature.turnoff", true); - configurationService.setProperty("dspace.server.ssr.url", "http://ssr.example.com/api"); + // Disarm the alwaysThrowExceptionFeature and define the SSR base URL via system properties so the + // overrides survive a mid-test config auto-reload (see enableSsrObjectResolution). + enableSsrObjectResolution(); String[] invalidUris = new String[] { "invalid-uri", "", @@ -1050,9 +1087,9 @@ public void findByObjectBadRequestSSRTest() throws Exception { public void findByNotExistingObjectSSSTest() throws Exception { String wrongSiteUri = "http://localhost/api/core/sites/" + UUID.randomUUID(); - // disarm the alwaysThrowExceptionFeature - configurationService.setProperty("org.dspace.app.rest.authorization.AlwaysThrowExceptionFeature.turnoff", true); - configurationService.setProperty("dspace.server.ssr.url", "http://ssr.example.com/api"); + // Disarm the alwaysThrowExceptionFeature and define the SSR base URL via system properties so the + // overrides survive a mid-test config auto-reload (see enableSsrObjectResolution). + enableSsrObjectResolution(); String adminToken = getAuthToken(admin.getEmail(), password); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamControllerIT.java index 63f7e62206f8..7fb2f4cecdf8 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamControllerIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamControllerIT.java @@ -7,17 +7,20 @@ */ package org.dspace.app.rest; +import static org.junit.Assert.assertEquals; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -import java.io.ByteArrayOutputStream; +import java.io.ByteArrayInputStream; import java.io.InputStream; -import java.util.zip.Deflater; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.apache.commons.codec.CharEncoding; -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; import org.apache.commons.io.IOUtils; import org.dspace.app.rest.model.ItemRest; import org.dspace.app.rest.test.AbstractControllerIntegrationTest; @@ -29,7 +32,6 @@ import org.dspace.content.Bitstream; import org.dspace.content.Collection; import org.dspace.content.Item; -import org.dspace.content.service.BitstreamService; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -38,6 +40,7 @@ public class MetadataBitstreamControllerIT extends AbstractControllerIntegration private static final String ALL_ZIP_PATH = "allzip"; private static final String HANDLE_PARAM = "handleId"; private static final String AUTHOR = "Test author name"; + private static final String BITSTREAM_CONTENT = "ThisIsSomeDummyText"; private Collection col; private Item publicItem; @@ -46,9 +49,6 @@ public class MetadataBitstreamControllerIT extends AbstractControllerIntegration @Autowired AuthorizeService authorizeService; - @Autowired - BitstreamService bitstreamService; - @Override public void setUp() throws Exception { @@ -64,7 +64,7 @@ public void setUp() throws Exception { .withAuthor(AUTHOR) .build(); - String bitstreamContent = "ThisIsSomeDummyText"; + String bitstreamContent = BITSTREAM_CONTENT; try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { bts = BitstreamBuilder. createBitstream(context, publicItem, is) @@ -78,23 +78,30 @@ public void setUp() throws Exception { @Test public void downloadAllZip() throws Exception { - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - ZipArchiveOutputStream zip = new ZipArchiveOutputStream(byteArrayOutputStream); - zip.setCreateUnicodeExtraFields(ZipArchiveOutputStream.UnicodeExtraFieldPolicy.ALWAYS); - zip.setLevel(Deflater.NO_COMPRESSION); - ZipArchiveEntry ze = new ZipArchiveEntry(bts.getName()); - zip.putArchiveEntry(ze); - InputStream is = bitstreamService.retrieve(context, bts); - org.apache.commons.compress.utils.IOUtils.copy(is, zip); - zip.closeArchiveEntry(); - is.close(); - zip.close(); - String token = getAuthToken(admin.getEmail(), password); - getClient(token).perform(get(METADATABITSTREAM_ENDPOINT + "/" + publicItem.getID() + + byte[] zipBytes = getClient(token).perform(get(METADATABITSTREAM_ENDPOINT + "/" + publicItem.getID() + "/" + ALL_ZIP_PATH).param(HANDLE_PARAM, publicItem.getHandle())) .andExpect(status().isOk()) - .andExpect(content().bytes(byteArrayOutputStream.toByteArray())); + .andReturn().getResponse().getContentAsByteArray(); + + // A ZIP entry stores a DOS last-modified timestamp that defaults to "now" at 2-second resolution, so + // comparing the response byte-for-byte against a locally-built ZIP intermittently failed when the server + // and the test happened to build their entries in different time buckets. Assert the meaningful payload + // instead: the archive must contain exactly the item's bitstream, with the expected content. + Map entries = new HashMap<>(); + int entryCount = 0; + try (ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(zipBytes))) { + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + entryCount++; + entries.put(entry.getName(), new String(IOUtils.toByteArray(zis), StandardCharsets.UTF_8)); + zis.closeEntry(); + } + } + // count tracked separately so a duplicate entry name can't be masked by the map + assertEquals(1, entryCount); + assertEquals(Set.of(bts.getName()), entries.keySet()); + assertEquals(BITSTREAM_CONTENT, entries.get(bts.getName())); } } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/StatisticsRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/StatisticsRestRepositoryIT.java index 6989c208617d..716ccf8b8de6 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/StatisticsRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/StatisticsRestRepositoryIT.java @@ -938,6 +938,13 @@ public void topCountriesReport_Community_Visited() throws Exception { .contentType(contentType)) .andExpect(status().isCreated()); + // Force a commit that waits for a new searcher so the two just-posted view events are guaranteed to be + // flushed and visible to the report query below. This covers both ways the report could otherwise come + // back empty: postView()'s own commit uses waitSearcher=false (can return before the searcher reopens), + // and if the solr-statistics.autoCommit=false override were dropped by a mid-test config reload then + // postView() skips its commit entirely (Solr's own autoCommit only fires after 10s). + StatisticsServiceFactory.getInstance().getSolrLoggerService().commit(); + // And request that collection's TopCountries report getClient(adminToken).perform( get("/api/statistics/usagereports/" + communityVisited.getID() + "_" + TOP_COUNTRIES_REPORT_ID)) From 8320889f2cb9fd56a313665adbcf69655859e408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Fri, 26 Jun 2026 14:50:20 +0200 Subject: [PATCH 36/41] [Backport dtq-dev] Issue 1360: allow to change license in workflow item, in PATCH operation (ufal#1365) (#1327) * Issue 1360: allow to change license in workflow item, in PATCH operation (ufal/clarin-dspace#1365) * Issue 1360: allow to change license in workflow item, in PATCH operation * improve fix + Integration test * resolve Copilot comments * resolve more MR comments * return 404 in case PATCH request references non existing license * improve error message for task claimed by other user * resolve second round of Copilot comments * small code refactoring * fixed failing test (cherry picked from commit 40672c32418aef02cab6e5124d811d959886af6f) * resolve PR comments --------- Co-authored-by: Milan Kuchtiak --- .../WorkflowItemRestRepository.java | 82 +++++++++- .../ClarinWorkflowItemRestRepositoryIT.java | 153 ++++++++++++++++++ .../app/rest/TaskRestRepositoriesIT.java | 5 +- 3 files changed, 230 insertions(+), 10 deletions(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkflowItemRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkflowItemRestRepository.java index 022064576ea1..a7dd352cec4e 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkflowItemRestRepository.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/WorkflowItemRestRepository.java @@ -7,14 +7,18 @@ */ package org.dspace.app.rest.repository; +import static org.dspace.app.rest.repository.ClarinLicenseRestRepository.OPERATION_PATH_LICENSE_RESOURCE; import static org.dspace.xmlworkflow.state.actions.processingaction.ProcessingAction.SUBMIT_EDIT_METADATA; import java.io.IOException; import java.sql.SQLException; import java.util.List; +import java.util.Objects; import java.util.UUID; import javax.servlet.http.HttpServletRequest; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.dspace.app.rest.Parameter; @@ -24,9 +28,12 @@ import org.dspace.app.rest.exception.UnprocessableEntityException; import org.dspace.app.rest.model.ErrorRest; import org.dspace.app.rest.model.WorkflowItemRest; +import org.dspace.app.rest.model.patch.JsonValueEvaluator; import org.dspace.app.rest.model.patch.Operation; import org.dspace.app.rest.model.patch.Patch; +import org.dspace.app.rest.model.patch.ReplaceOperation; import org.dspace.app.rest.submit.SubmissionService; +import org.dspace.app.rest.submit.step.ClarinLicenseSubmissionUtils; import org.dspace.app.rest.utils.SolrOAIReindexer; import org.dspace.app.util.SubmissionConfigReaderException; import org.dspace.authorize.AuthorizeException; @@ -200,12 +207,16 @@ public Class getDomainClass() { @Override public WorkflowItemRest upload(HttpServletRequest request, String apiCategory, String model, Integer id, - MultipartFile file) throws SQLException { + MultipartFile file) throws SQLException, AuthorizeException { Context context = obtainContext(); WorkflowItemRest wsi = findOne(context, id); XmlWorkflowItem source = wis.find(context, id); + if (source == null) { + throw new ResourceNotFoundException("WorkflowItem with id " + id + " not found"); + } + this.checkIfEditMetadataAllowedInCurrentStep(context, source); List errors = submissionService.uploadFileToInprogressSubmission(context, request, wsi, source, file); @@ -226,17 +237,27 @@ public void patch(Context context, HttpServletRequest request, String apiCategor WorkflowItemRest wsi = findOne(context, id); XmlWorkflowItem source = wis.find(context, id); + if (source == null) { + throw new ResourceNotFoundException("WorkflowItem with id " + id + " not found"); + } + this.checkIfEditMetadataAllowedInCurrentStep(context, source); for (Operation op : operations) { //the value in the position 0 is a null value String[] path = op.getPath().substring(1).split("/", 3); - if (OPERATION_PATH_SECTIONS.equals(path[0])) { + if (OPERATION_PATH_LICENSE_RESOURCE.equals(path[0])) { + // Apply the CLARIN license change through the shared submission helper so the + // workflow `/license` path behaves the same as the submission license paths. + // A non-existing license surfaces as ClarinLicenseNotFoundException (404). + ClarinLicenseSubmissionUtils.applyLicense(context, source.getItem(), extractLicenseName(op)); + } else if (OPERATION_PATH_SECTIONS.equals(path[0])) { String section = path[1]; submissionService.evaluatePatchToInprogressSubmission(context, request, source, wsi, section, op); } else { throw new DSpaceBadRequestException( - "Patch path operation need to starts with '" + OPERATION_PATH_SECTIONS + "'"); + "Patch path operation need to starts with '" + + OPERATION_PATH_LICENSE_RESOURCE + "' or '" + OPERATION_PATH_SECTIONS + "'"); } } wis.update(context, source); @@ -278,20 +299,65 @@ protected void delete(Context context, Integer id) { } } + /** + * Extract the CLARIN license name from a JSON Patch {@code replace} operation on the {@code /license} + * path. The value is accepted either as a plain string or as an object wrapping a textual {@code value} + * field; a non-replace operation or any other value shape is rejected as a bad request. A blank name is + * passed through (the submission helper treats it as a request to clear the current license selection). + * @param op the JSON Patch operation targeting the license path + * @return the CLARIN license name to apply + */ + private String extractLicenseName(Operation op) { + if (!(op instanceof ReplaceOperation)) { + throw new DSpaceBadRequestException("The operation to update the license must be the 'replace' operation"); + } + if (op.getValue() instanceof String) { + return (String) op.getValue(); + } + if (!(op.getValue() instanceof JsonValueEvaluator)) { + throw wrongValueFormatException(op); + } + JsonValueEvaluator jsonValEvaluator = (JsonValueEvaluator) op.getValue(); + if (!(jsonValEvaluator.getValueNode() instanceof ObjectNode)) { + throw wrongValueFormatException(op); + } + // a replace operation may wrap the value in an ObjectNode under the "value" key + JsonNode jsonNodeValue = jsonValEvaluator.getValueNode().get("value"); + if (jsonNodeValue != null && jsonNodeValue.isTextual()) { + return jsonNodeValue.asText(); + } + throw wrongValueFormatException(op); + } + + private DSpaceBadRequestException wrongValueFormatException(Operation op) { + return new DSpaceBadRequestException("Unsupported value format for operation '" + op.getOp() + + "'. Expected a string or an object with a textual 'value' field."); + } + /** * Checks if @link{SUBMIT_EDIT_METADATA} is a valid option in the workflow step this task is currently at. * Patching and uploading is only allowed if this is the case. * @param context Context * @param xmlWorkflowItem WorkflowItem of the task */ - private void checkIfEditMetadataAllowedInCurrentStep(Context context, XmlWorkflowItem xmlWorkflowItem) { + private void checkIfEditMetadataAllowedInCurrentStep(Context context, XmlWorkflowItem xmlWorkflowItem) + throws AuthorizeException { try { - ClaimedTask claimedTask = claimedTaskService.findByWorkflowIdAndEPerson(context, xmlWorkflowItem, - context.getCurrentUser()); - if (claimedTask == null) { + List claimTasks = claimedTaskService.findByWorkflowItem(context, xmlWorkflowItem); + if (claimTasks.isEmpty()) { throw new UnprocessableEntityException("WorkflowItem with id " + xmlWorkflowItem.getID() - + " has not been claimed yet."); + + " has not been claimed yet."); } + + ClaimedTask claimedTask = claimTasks.stream() + .filter(ct -> Objects.equals(ct.getOwner(), context.getCurrentUser())) + .findFirst() + .orElse(null); + if (claimedTask == null) { + throw new AuthorizeException("The current user hasn't claimed the workflow item with id " + + xmlWorkflowItem.getID() + ", so the user cannot patch this item"); + } + Workflow workflow = workflowFactory.getWorkflow(claimedTask.getWorkflowItem().getCollection()); Step step = workflow.getStep(claimedTask.getStepID()); WorkflowActionConfig currentActionConfig = step.getActionConfig(claimedTask.getActionID()); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinWorkflowItemRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinWorkflowItemRestRepositoryIT.java index 6c2d84f76aa7..47fcc2b729a5 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinWorkflowItemRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinWorkflowItemRestRepositoryIT.java @@ -15,16 +15,29 @@ import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertFalse; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.lang3.StringUtils; +import org.dspace.app.rest.model.patch.AddOperation; +import org.dspace.app.rest.model.patch.Operation; +import org.dspace.app.rest.model.patch.ReplaceOperation; +import org.dspace.app.rest.repository.ClarinLicenseRestRepository; import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.builder.ClaimedTaskBuilder; +import org.dspace.builder.ClarinLicenseBuilder; +import org.dspace.builder.ClarinLicenseLabelBuilder; import org.dspace.builder.CollectionBuilder; import org.dspace.builder.CommunityBuilder; import org.dspace.builder.EPersonBuilder; @@ -35,12 +48,18 @@ import org.dspace.content.Item; import org.dspace.content.MetadataValue; import org.dspace.content.WorkspaceItem; +import org.dspace.content.clarin.ClarinLicense; +import org.dspace.content.clarin.ClarinLicenseLabel; import org.dspace.content.service.ItemService; import org.dspace.content.service.WorkspaceItemService; +import org.dspace.content.service.clarin.ClarinLicenseLabelService; +import org.dspace.content.service.clarin.ClarinLicenseService; import org.dspace.eperson.EPerson; import org.dspace.license.service.CreativeCommonsService; import org.dspace.services.ConfigurationService; import org.dspace.xmlworkflow.factory.XmlWorkflowFactory; +import org.dspace.xmlworkflow.storedcomponents.ClaimedTask; +import org.dspace.xmlworkflow.storedcomponents.XmlWorkflowItem; import org.dspace.xmlworkflow.storedcomponents.service.CollectionRoleService; import org.dspace.xmlworkflow.storedcomponents.service.XmlWorkflowItemService; import org.hamcrest.Matchers; @@ -78,6 +97,11 @@ public class ClarinWorkflowItemRestRepositoryIT extends AbstractControllerIntegr @Autowired private ItemService itemService; + @Autowired + private ClarinLicenseService clarinLicenseService; + @Autowired + private ClarinLicenseLabelService clarinLicenseLabelService; + Item item; @Before @@ -364,4 +388,133 @@ public void shouldCreateItemWithCustomTypeBindField() throws Exception { assertFalse(mvList.isEmpty()); assertThat(mvList.get(0).getValue(), is(CITATION_VALUE)); } + + @Test + public void patchUpdateClarinLicense() throws Exception { + context.turnOffAuthorisationSystem(); + + // create Clarin License Label + ClarinLicenseLabel clarinLicenseLabel = ClarinLicenseLabelBuilder.createClarinLicenseLabel(context).build(); + clarinLicenseLabel.setLabel("CC"); + clarinLicenseLabel.setExtended(false); + clarinLicenseLabel.setTitle("CLL Title1"); + clarinLicenseLabelService.update(context, clarinLicenseLabel); + + // create Clarin License + ClarinLicense clarinLicense = ClarinLicenseBuilder.createClarinLicense(context).build(); + clarinLicense.setName("CL Name"); + clarinLicense.setConfirmation(ClarinLicense.Confirmation.NOT_REQUIRED); + clarinLicense.setDefinition("CL Definition"); + clarinLicense.setRequiredInfo("CL Req"); + // add clarinLicenseLabel to clarinLicense + Set clarinLicenseLabels = new HashSet<>(); + clarinLicenseLabels.add(clarinLicenseLabel); + clarinLicense.setLicenseLabels(clarinLicenseLabels); + clarinLicenseService.update(context, clarinLicense); + + // community with one collection. + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + Collection col = CollectionBuilder.createCollection(context, parentCommunity).withName("Collection 1") + .withWorkflowGroup("editor", eperson).build(); + + // create a normal user to use as submitter + EPerson submitter = EPersonBuilder.createEPerson(context) + .withEmail("submitter@example.com") + .withPassword("dspace") + .build(); + + // claimed task with workflow item in edit step + ClaimedTask claimedTask = ClaimedTaskBuilder.createClaimedTask(context, col, eperson) + .withTitle("Workflow Item") + .withIssueDate("2026-05-18") + .withSubject("Extra Entry") + .grantLicense() + .build(); + claimedTask.setStepID("editstep"); + claimedTask.setActionID("editaction"); + XmlWorkflowItem wfItem = claimedTask.getWorkflowItem(); + + context.restoreAuthSystemState(); + + // prepare a patch targeting the clarin license resource path + List ops = new ArrayList<>(); + ops.add(new ReplaceOperation("/" + ClarinLicenseRestRepository.OPERATION_PATH_LICENSE_RESOURCE, "CL Name")); + + String submitterToken = getAuthToken(submitter.getEmail(), "dspace"); + + // The submitter shouldn't be allowed to patch clarin license + // because the workflow item was claimed by the other user (eperson), + // and the submitter doesn't have permissions to edit it, + // so the patch request should be rejected with error 403(Forbidden) + getClient(submitterToken).perform(patch("/api/workflow/workflowitems/" + wfItem.getID()) + .content(getPatchContent(ops)) + .contentType(javax.ws.rs.core.MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isForbidden()); + + String editorToken = getAuthToken(eperson.getEmail(), password); + + ops.set(0, new ReplaceOperation("/" + ClarinLicenseRestRepository.OPERATION_PATH_LICENSE_RESOURCE, "Wrong CL")); + + // The wrong clarin license name value should be rejected with 404 Not Found + getClient(editorToken).perform(patch("/api/workflow/workflowitems/" + wfItem.getID()) + .content(getPatchContent(ops)) + .contentType(javax.ws.rs.core.MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isNotFound()); + + // The valid clarin license name can be in the form of a simple string or + // in the form of a map with "value" key, but it should be accepted in both cases + ops.set(0, new ReplaceOperation("/" + ClarinLicenseRestRepository.OPERATION_PATH_LICENSE_RESOURCE, "CL Name")); + + getClient(editorToken).perform(patch("/api/workflow/workflowitems/" + wfItem.getID()) + .content(getPatchContent(ops)) + .contentType(javax.ws.rs.core.MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isOk()); + + XmlWorkflowItem updatedWfItem = xmlWorkflowItemService.find(context, wfItem.getID()); + assertThat(itemService.getMetadataFirstValue(updatedWfItem.getItem(), "dc", "rights", null, Item.ANY), + is("CL Name")); + + Map wrappedValue = new HashMap(); + wrappedValue.put("value", "CL Name"); + ops.set(0, new ReplaceOperation("/" + ClarinLicenseRestRepository.OPERATION_PATH_LICENSE_RESOURCE, + wrappedValue)); + + getClient(editorToken).perform(patch("/api/workflow/workflowitems/" + wfItem.getID()) + .content(getPatchContent(ops)) + .contentType(javax.ws.rs.core.MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isOk()); + + // The wrapped value should contain the "value" key, otherwise it is invalid + Map invalidWrappedValue1 = new HashMap(); + ops.set(0, new ReplaceOperation("/" + ClarinLicenseRestRepository.OPERATION_PATH_LICENSE_RESOURCE, + invalidWrappedValue1)); + + getClient(editorToken).perform(patch("/api/workflow/workflowitems/" + wfItem.getID()) + .content(getPatchContent(ops)) + .contentType(javax.ws.rs.core.MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isBadRequest()); + + // The wrapped value should be in a map, not in a list + List invalidWrappedValue2 = new ArrayList<>(); + invalidWrappedValue2.add("CL Name"); + ops.set(0, new ReplaceOperation("/" + ClarinLicenseRestRepository.OPERATION_PATH_LICENSE_RESOURCE, + invalidWrappedValue2)); + + getClient(editorToken).perform(patch("/api/workflow/workflowitems/" + wfItem.getID()) + .content(getPatchContent(ops)) + .contentType(javax.ws.rs.core.MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isBadRequest()); + + // The only accepted operation for clarin license resource is "replace", + // "add" operation should be rejected with 400 Bad Request even with the valid value + ops.set(0, new AddOperation("/" + ClarinLicenseRestRepository.OPERATION_PATH_LICENSE_RESOURCE, + "CL Name")); + + getClient(editorToken).perform(patch("/api/workflow/workflowitems/" + wfItem.getID()) + .content(getPatchContent(ops)) + .contentType(javax.ws.rs.core.MediaType.APPLICATION_JSON_PATCH_JSON)) + .andExpect(status().isBadRequest()); + } } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/TaskRestRepositoriesIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/TaskRestRepositoriesIT.java index a9b5c6a582b6..31c0a8e1e289 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/TaskRestRepositoriesIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/TaskRestRepositoriesIT.java @@ -2879,7 +2879,8 @@ public void patchTest_ClaimedTask_EditMetadataOptionNotAllowed() throws Exceptio .andExpect(status().isCreated()) .andExpect(jsonPath("$", Matchers.allOf(hasJsonPath("$.type", is("claimedtask"))))); - // try to patch a workspace item while it is in a step that does not have the edit_metadata option (review step) + // try to patch a workflow item by a user who does not have the edit metadata permission + // in the current step (review step) String authToken = getAuthToken(eperson.getEmail(), password); // a simple patch to update an existent metadata @@ -2893,7 +2894,7 @@ public void patchTest_ClaimedTask_EditMetadataOptionNotAllowed() throws Exceptio getClient(authToken).perform(patch("/api/workflow/workflowitems/" + witem.getID()) .content(patchBody) .contentType(javax.ws.rs.core.MediaType.APPLICATION_JSON_PATCH_JSON)) - .andExpect(status().isUnprocessableEntity()); + .andExpect(status().isForbidden()); } @Test From 6e0ed0c6ccdcab2451acfa88739d4dbd3a58fa99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Fri, 26 Jun 2026 14:52:53 +0200 Subject: [PATCH 37/41] [Port to dtq-dev] Issue ufal/clarin-dspace#1317 metadata health check (#1307) * Issue ufal/clarin-dspace#1317 metadata health check (ufal/clarin-dspace#1338) * Issue 1317: health-check for metadata - initial commit * implement MetadataCheck report * improve MetadataCheck * improve MetadataCheck with more sophisticated selection of error/warning messages * code cleanup * rename qa-metadata-error-patterns.json to metadata-check-patterns.json * implement copilot suggestions * add test for Metadata check to HealthReportIT * improved documentation * added test * improve documentation * set default error dispersion quota to 5 (cherry picked from commit 39157a5cc54f41bb6a0a06569859fcd4ae70fc2e) * add JavaDoc description * resolve coderabbitai comment * update report-diff-fields.json * resolve Copilot comments --------- Co-authored-by: Milan Kuchtiak --- .../java/org/dspace/health/MetadataCheck.java | 495 ++++++++++++++++++ .../resources/metadata-check-patterns.json | 52 ++ .../main/resources/report-diff-fields.json | 9 +- .../org/dspace/scripts/HealthReportIT.java | 243 +++++++++ dspace/config/modules/healthcheck.cfg | 6 +- 5 files changed, 801 insertions(+), 4 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/health/MetadataCheck.java create mode 100644 dspace-api/src/main/resources/metadata-check-patterns.json diff --git a/dspace-api/src/main/java/org/dspace/health/MetadataCheck.java b/dspace-api/src/main/java/org/dspace/health/MetadataCheck.java new file mode 100644 index 000000000000..f3793e6a9356 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/health/MetadataCheck.java @@ -0,0 +1,495 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.health; + +import java.io.IOException; +import java.io.InputStream; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; +import java.util.stream.StreamSupport; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.commons.collections.ListUtils; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.core.Context; +import org.dspace.curate.Curator; +import org.dspace.services.ConfigurationService; +import org.dspace.utils.DSpace; +import org.json.JSONArray; +import org.json.JSONObject; + +/** + * This check runs the "metadataqa" curation task on the whole repository, + * and provides a report about the number of errors and warnings. + * + * @author Milan Kuchtiak + */ +public class MetadataCheck extends Check { + + private static final String CURATION_TASK_NAME = "metadataqa"; + private static final String QA_METADATA_ERROR_PATTERNS_JSON = "metadata-check-patterns.json"; + private static final String VALIDATION_TYPE_OTHER = "validation.other"; + private static final int COUNT_INDENTATION = 30; + + // default values for configuration properties, which can be overridden in configuration + + // the maximum number of errors to be shown in the report + private static final int MAXIMUM_ERRORS_TO_SHOW = 100; + // the maximum number of warnings to be shown in the report + private static final int MAXIMUM_WARNINGS_TO_SHOW = 50; + // This number is only relevant when the number of errors exceeds the maximum number of errors to be shown. + // Represents the dispersion of the error messages. + // The frequency of the new (upcoming) message in the report is compared with the frequency of the + // most frequent message in the report, and the replacement is made when the frequency of the new message + // is significantly lower than the frequency of the most frequent message + // (when the difference in occurrence is higher than the dispersion quota). + private static final int ERROR_DISPERSION_QUOTA = 5; + // the same as ERROR_DISPERSION_QUOTA but for warnings + private static final int WARNING_DISPERSION_QUOTA = 5; + + private static Map> errorPatterns; + private static Map> warningPatterns; + + static { + try { + loadPatterns(); + } catch (IOException e) { + throw new RuntimeException("Cannot load error patterns", e); + } + } + + @Override + public String run(ReportInfo ri) { + + ConfigurationService configurationService = new DSpace().getConfigurationService(); + + int maxErrorsToShow = configurationService.getIntProperty("healthcheck.metadata.max-errors-to-show", + MAXIMUM_ERRORS_TO_SHOW); + + int maxWarningsToShow = configurationService.getIntProperty("healthcheck.metadata.max-warnings-to-show", + MAXIMUM_WARNINGS_TO_SHOW); + + int errorDispersionQuota = configurationService.getIntProperty("healthcheck.metadata.error-dispersion-quota", + ERROR_DISPERSION_QUOTA); + + int warningDispersionQuota = + configurationService.getIntProperty("healthcheck.metadata.warning-dispersion-quota", + WARNING_DISPERSION_QUOTA); + + StringBuilder sb = new StringBuilder(); + JSONObject root = new JSONObject(); + + Curator curator = new Curator(); + curator.addTask(CURATION_TASK_NAME); + + MetadataReporter reporter = new MetadataReporter( + maxErrorsToShow, + maxWarningsToShow, + errorDispersionQuota, + warningDispersionQuota); + + curator.setReporter(reporter); + try (Context context = new Context()) { + curator.curate(context, ContentServiceFactory.getInstance().getSiteService().findSite(context).getHandle()); + context.complete(); + } catch (IOException | SQLException e) { + error(e, "Error during curation"); + } + + Map errorCounts = reporter.getErrorCount(); + int overallErrorCount = errorCounts.values().stream().mapToInt(Integer::intValue).sum(); + + Map warningCounts = reporter.getWarningCount(); + int overallWarningCount = warningCounts.values().stream().mapToInt(Integer::intValue).sum(); + + Map> errorMessages = reporter.getErrorMessages(); + Map> warningMessages = reporter.getWarningMessages(); + + // error statistics + if (overallErrorCount > 0) { + sb.append("\nError statistics:\n\n"); + errorCounts.forEach((key, val) -> { + String errorCode = formatErrorCode(key); + sb.append(errorCode).append(" ".repeat(COUNT_INDENTATION - errorCode.length())) + .append(String.format("%7d", val)).append("\n"); + }); + sb.append("-".repeat(COUNT_INDENTATION + 7)).append("\n"); + sb.append("Error count total: ") + .append(" ".repeat(COUNT_INDENTATION - "Error count total: ".length())) + .append(String.format("%7d", overallErrorCount)).append("\n"); + } + + // warning statistics + if (overallWarningCount > 0) { + sb.append("\nWarning statistics:\n\n"); + warningCounts.forEach((key, val) -> { + String errorCode = formatErrorCode(key); + sb.append(errorCode).append(" ".repeat(COUNT_INDENTATION - errorCode.length())) + .append(String.format("%7d", val)).append("\n"); + }); + sb.append("-".repeat(COUNT_INDENTATION + 7)).append("\n"); + sb.append("Warning count total: ") + .append(" ".repeat(COUNT_INDENTATION - "Warning count total: ".length())) + .append(String.format("%7d", overallWarningCount)).append("\n"); + } + + // list of errors + if (overallErrorCount > 0) { + sb.append("\nErrors:\n"); + errorMessages.forEach((key, messages) -> + messages.forEach(message -> sb.append(message).append("\n")) + ); + if (overallErrorCount > maxErrorsToShow) { + sb.append("and more...\n"); + } + } + + // list of warnings + if (overallWarningCount > 0) { + sb.append("\nWarnings:\n"); + warningMessages.forEach((key, messages) -> + messages.forEach(message -> sb.append(message).append("\n")) + ); + if (overallWarningCount > maxWarningsToShow) { + sb.append("and more...\n"); + } + } + + // populate JSON report + root.put("errorCount", overallErrorCount); + root.put("warningCount", overallWarningCount); + + JSONArray errors = new JSONArray(); + errorCounts.forEach((key, val) -> { + JSONObject error = new JSONObject() + .put("type", key) + .put("count", val); + errors.put(error); + }); + root.put("errors", errors); + + JSONArray warnings = new JSONArray(); + warningCounts.forEach((key, val) -> { + JSONObject warning = new JSONObject() + .put("type", key) + .put("count", val); + warnings.put(warning); + }); + root.put("warnings", warnings); + + this.setReportJson(root); + return sb.toString(); + } + + private static void loadPatterns() throws IOException { + try (InputStream qaMetadataErrors = Thread.currentThread() + .getContextClassLoader().getResourceAsStream(QA_METADATA_ERROR_PATTERNS_JSON);) { + if (qaMetadataErrors == null) { + throw new IOException("Resource '" + QA_METADATA_ERROR_PATTERNS_JSON + + "' not found in classpath"); + } + JsonNode root = new ObjectMapper().readTree(qaMetadataErrors); + + // Load error types and their associated error patterns + errorPatterns = getPatterns(root.withObject("errors")); + // Load warning types and their associated warning patterns + warningPatterns = getPatterns(root.withObject("warnings")); + } + } + + private static Map> getPatterns(JsonNode parentNode) { + Map> validationPatterns = new HashMap<>(); + parentNode.fieldNames().forEachRemaining(validationType -> { + List validationMessages = new ArrayList<>(); + ArrayNode patterns = parentNode.withArray(validationType); + StreamSupport.stream(patterns.spliterator(), false).forEach(message -> { + validationMessages.add(message.asText()); + }); + validationPatterns.put(validationType, validationMessages); + }); + + return validationPatterns; + } + + private static String formatErrorCode(String errorCode) { + if (errorCode.startsWith("validation.")) { + String validationCode = errorCode.substring("validation.".length()); + return validationCode.replaceAll("\\.", " ") + " issues: "; + } else { + return errorCode + " issues: "; + } + } + + private static class MetadataReporter implements Appendable { + + private final int maxErrorsToShow; + private final int maxWarningsToShow; + private final int errorDispersionQuota; + private final int warningDispersionQuota; + + private final Map errorCount = new TreeMap<>(); + private final Map warningCount = new TreeMap<>(); + + // represent stored messages for errors and warnings + private final StoredMessagesInfo errorMessages = new StoredMessagesInfo(); + private final StoredMessagesInfo warningMessages = new StoredMessagesInfo(); + + Map> getErrorMessages() { + return errorMessages.getStoredMessages(); + } + + Map getErrorCount() { + return errorCount; + } + + Map> getWarningMessages() { + return warningMessages.getStoredMessages(); + } + + Map getWarningCount() { + return warningCount; + } + + MetadataReporter(int maxErrorsToShow, + int maxWarningsToShow, + int errorDispersionQuota, + int warningDispersionQuota) { + this.maxErrorsToShow = maxErrorsToShow; + this.maxWarningsToShow = maxWarningsToShow; + this.errorDispersionQuota = errorDispersionQuota; + this.warningDispersionQuota = warningDispersionQuota; + } + + @Override + public Appendable append(CharSequence cs) throws IOException { + String line = cs.toString(); + if (line.contains("ERROR! ")) { + populateData( + "ERROR! ", + line, + errorPatterns, + errorCount, + errorMessages, + maxErrorsToShow, + errorDispersionQuota + ); + } else if (line.contains("Warning: ")) { + populateData( + "Warning: ", + line, + warningPatterns, + warningCount, + warningMessages, + maxWarningsToShow, + warningDispersionQuota + ); + } + return this; + } + + @Override + public Appendable append(CharSequence cs, int i, int i1) throws IOException { + return this.append(cs.subSequence(i, i1)); + } + + @Override + public Appendable append(char c) throws IOException { + return this.append(String.valueOf(c)); + } + + private void populateData(String prefix, + String line, + Map> patterns, + Map counts, + StoredMessagesInfo storedMessagesInfo, + int limit, + int dispersionQuota + ) { + int startIndex = line.indexOf(prefix) + prefix.length(); + String fullMessage = line.substring(startIndex); + int endIndex = fullMessage.lastIndexOf("[["); + String messageKey; + if (endIndex > 0) { + messageKey = fullMessage.substring(0, endIndex - 1); + } else { + messageKey = fullMessage; + } + Message message = new Message(messageKey, fullMessage); + boolean found = false; + for (Map.Entry> entry : patterns.entrySet()) { + String type = entry.getKey(); + List typePatterns = entry.getValue(); + for (String pattern : typePatterns) { + boolean startsWithCaret = pattern.startsWith("^"); + boolean endsWithDollar = pattern.endsWith("$"); + if ((startsWithCaret && messageKey.startsWith(pattern.substring(1))) || + (endsWithDollar && messageKey.endsWith(pattern.substring(0, pattern.length() - 1))) || + (!startsWithCaret && !endsWithDollar && messageKey.contains(pattern)) + ) { + addMessage(type, message, counts, storedMessagesInfo, limit, dispersionQuota); + found = true; + break; + } + } + if (found) { + break; // If a pattern is found, no need to check other patterns for this message + } + } + if (!found) { + // If no pattern matched, categorize under "validation.other" + addMessage(VALIDATION_TYPE_OTHER, message, counts, storedMessagesInfo, limit, dispersionQuota); + } + } + + private void addMessage(String validationType, + Message message, + Map counts, + StoredMessagesInfo storedMessagesInfo, + int limit, + int dispersionQuota) { + // increase the count for this validation type + counts.merge(validationType, 1, Integer::sum); + int mCount = storedMessagesInfo.getCount(); + Map> messages = storedMessagesInfo.getStoredMessages(); + if (mCount < limit) { + // add error|warning to messages and increase the overall messages count + messages.merge(message.getMessageKey(), List.of(message.getFullMessage()), ListUtils::union); + storedMessagesInfo.count++; + } else { + // replace one of the stored messages with new message when possible + // but don't change the overall messages count + replaceMessage(message, storedMessagesInfo, dispersionQuota); + } + } + + /** + * Try to replace one of the stored messages, with the highest frequency, with the new message. + * The replacement is made when the new message is entirely new + * or the frequency of the new message is significantly lower than the messages with the highest frequency. + * + * @param message the new message that should be added to stored messages + * @param storedMessagesInfo the messages that are already stored for the report + * @param dispersionQuota quota saying how much of the messages with the highest frequency is acceptable to keep + * comparing to the frequency of the new message + */ + private void replaceMessage(Message message, StoredMessagesInfo storedMessagesInfo, int dispersionQuota) { + Map> storedMessages = storedMessagesInfo.getStoredMessages(); + + // calculate the highest frequency of messages for any short message in stored messages, + String messageKeyWithHighestFrequency = Objects.requireNonNull(getMessageWithHighestCount(storedMessages)); + int highestMessageFrequency = storedMessages.get(messageKeyWithHighestFrequency).size(); + + // int highestMessageFrequency = storedMessagesInfo.getHighestFrequency(); + if (highestMessageFrequency <= 1) { + // no replacement, as there are no messages with the frequency higher than 1, so the replacement + // of any message would not increase the diversity of messages in stored messages + return; + } + String messageKey = message.getMessageKey(); + List storedMessagesForMessageKey = storedMessages.get(messageKey); + + if (storedMessagesForMessageKey != null && + (storedMessagesForMessageKey.size() + dispersionQuota >= highestMessageFrequency)) { + // no replacement, as the frequency of the new message is not significantly lower + // than the frequency of the message with the highest frequency + return; + } + + // recalculate the highest frequency of messages for any short message in stored messages, + // because it can be changed after each replacement + messageKeyWithHighestFrequency = Objects.requireNonNull(getMessageWithHighestCount(storedMessages)); + highestMessageFrequency = storedMessages.get(messageKeyWithHighestFrequency).size(); + + if (highestMessageFrequency <= 1) { + // no replacement, as there are no messages with the frequency higher than 1 anymore + // (after the recalculation) + return; + } + + if (storedMessagesForMessageKey == null || + (storedMessagesForMessageKey.size() + dispersionQuota < highestMessageFrequency)) { + // either (1) message key is not present in stored messages yet, + // so the last stored message with the highest frequency is removed and this new message is added + // + // or (2) message key is present in stored messages, + // but the frequency of messages for this message key is much lower + // than the frequency of other stored messages, + // so the (last) stored message with the highest frequency is removed and this new message is added + storedMessages.get(messageKeyWithHighestFrequency).remove(highestMessageFrequency - 1); + storedMessages.merge(messageKey, List.of(message.getFullMessage()), ListUtils::union); + } + } + + private static String getMessageWithHighestCount(Map> storedMessages) { + return storedMessages.entrySet() + .stream() + .max((e1, e2) -> Integer.compare(e1.getValue().size(), e2.getValue().size())) + .map(Map.Entry::getKey) + .orElseThrow(); + } + } + + /** + * Abstraction of the stored messages for errors and warnings, which are stored during the processing of messages + * and then used to generate the final report. + * The messages are stored in te form of a map, where the key is the message_key and + * the value is the list of full messages stored for this message_key. + * The count represents the overall number of stored messages. + * + * Example of message_key: "value [dc.date.available] is present multiple times" + * Example of the list of full messages: + * [ + * "value [dc.date.available] is present multiple times [[http://hdl.handle.net/123456789/2-7371]]", + * "value [dc.date.available] is present multiple times [[http://hdl.handle.net/123456789/2-7373]]", + * "value [dc.date.available] is present multiple times [[http://hdl.handle.net/123456789/2-7375]]" + * ] + * + */ + private static class StoredMessagesInfo { + private int count; + private final Map> storedMessages; + + StoredMessagesInfo() { + this.count = 0; + storedMessages = new TreeMap<>(); + } + + public int getCount() { + return count; + } + + public Map> getStoredMessages() { + return storedMessages; + } + } + + private static class Message { + private final String messageKey; + private final String fullMessage; + + public Message(String messageKey, String fullMessage) { + this.messageKey = messageKey; + this.fullMessage = fullMessage; + } + + public String getMessageKey() { + return messageKey; + } + + public String getFullMessage() { + return fullMessage; + } + } +} diff --git a/dspace-api/src/main/resources/metadata-check-patterns.json b/dspace-api/src/main/resources/metadata-check-patterns.json new file mode 100644 index 000000000000..fb2ba7329276 --- /dev/null +++ b/dspace-api/src/main/resources/metadata-check-patterns.json @@ -0,0 +1,52 @@ +{ + "errors": { + "dc.type": [ + "^Does not have dc.type metadata", + "^dc.type has null value", + "^leading or trailing spaces", + "^empty value", + "^invalid type (" + ], + "dc.language": [ + "^dc.language.iso", + "^Invalid language code", + "^local.language.name" + ], + "dc.title": [ + "^Item has no dc.title metadata", + "^Title " + ], + "dc.relation": [ + "^contains 'dc.relation.", + "^the referenced item" + ], + "dc.rights": [ + "^has labels ", + "^There are bitstreams but incomplete rights metadata." + ], + "dc.description": [ + "^contains suspicious [dc.description.uri] metadata" + ], + "local.branding": [ + "^local.branding " + ], + "validation.duplicate.value": [ + "is present multiple times$" + ], + "validation.missing.handle": [ + "^Does not have a handle" + ], + "validation.empty.value": [ + " is empty$", + " is null$" + ], + "validation.complex.type": [ + " is a component with " + ] + }, + "warnings": { + "dc.subject": [ + "^does not contain any [dc.subject] values" + ] + } +} \ No newline at end of file diff --git a/dspace-api/src/main/resources/report-diff-fields.json b/dspace-api/src/main/resources/report-diff-fields.json index 23891359a95e..8d98d10d6592 100644 --- a/dspace-api/src/main/resources/report-diff-fields.json +++ b/dspace-api/src/main/resources/report-diff-fields.json @@ -23,7 +23,10 @@ "/checks/[name=User summary]/report/subscribers": "Subscribers", "/checks/[name=User summary]/report/subscribedCollections": "Subscribed Collections", "/checks/[name=User summary]/report/emptyGroups": "Empty Groups", - "/checks/[name=License summary]/report/licenses": "Licenses" + "/checks/[name=License summary]/report/licenses": "Licenses", + "/checks/[name=Metadata check]/report/errorCount": "Metadata Errors", + "/checks/[name=Metadata check]/report/warningCount": "Metadata Warnings" + }, "fieldOrder": [ "/checks/[name=General Information]/report/directoryStats/0/size_bytes", @@ -49,6 +52,8 @@ "/checks/[name=User summary]/report/subscribers", "/checks/[name=User summary]/report/subscribedCollections", "/checks/[name=User summary]/report/emptyGroups", - "/checks/[name=License summary]/report/licenses" + "/checks/[name=License summary]/report/licenses", + "/checks/[name=Metadata check]/report/errorCount", + "/checks/[name=Metadata check]/report/warningCount" ] } \ No newline at end of file diff --git a/dspace-api/src/test/java/org/dspace/scripts/HealthReportIT.java b/dspace-api/src/test/java/org/dspace/scripts/HealthReportIT.java index 68142e18fcc4..219124a3970c 100644 --- a/dspace-api/src/test/java/org/dspace/scripts/HealthReportIT.java +++ b/dspace-api/src/test/java/org/dspace/scripts/HealthReportIT.java @@ -8,6 +8,8 @@ package org.dspace.scripts; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasItem; @@ -22,6 +24,10 @@ import java.util.List; import java.util.Set; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.commons.lang3.StringUtils; import org.dspace.AbstractIntegrationTestWithDatabase; import org.dspace.app.healthreport.HealthReport; import org.dspace.app.launcher.ScriptLauncher; @@ -47,6 +53,8 @@ import org.dspace.content.service.clarin.ClarinLicenseResourceMappingService; import org.dspace.content.service.clarin.ClarinLicenseService; import org.dspace.core.Constants; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; import org.junit.Test; /** @@ -274,4 +282,239 @@ public void testStoredArgsContainAllCheckOptions() throws Exception { assertThat(latest.getArgs(), containsString("-c: 2")); assertThat(latest.getArgs(), containsString("-c: 3")); } + + @Test + public void testMetadataCheck() throws Exception { + context.turnOffAuthorisationSystem(); + + Community community = CommunityBuilder.createCommunity(context) + .withName("Community") + .build(); + + Collection collection = CollectionBuilder.createCollection(context, community) + .withName("Collection") + .withSubmitterGroup(eperson) + .build(); + + Item item1 = ItemBuilder.createItem(context, collection) + .withTitle("Test item 1") + .withType("corpus") + .withMetadata("local", "branding", null, "Community") + .build(); + + Item item2 = ItemBuilder.createItem(context, collection) + .withTitle("Test item 2") + .withType("toolService") + .withSubject("Test subject") + .withMetadata("local", "branding", null, "Community") + .withMetadata("dc", "relation", "replaces", findItemUri(item1)) + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test item 3") + .withType("toolService") + .withSubject("Test subject") + .withMetadata("local", "branding", null, "Community") + .withMetadata("dc", "relation", "isreplacedby", findItemUri(item2)) + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test item 4") + .withMetadata("local", "branding", null, "Community") + .build(); + + ItemBuilder.createItem(context, collection) + .withType("toolService") + .withMetadata("local", "branding", null, "Community") + .build(); + + TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); + + // with "health-report -c 5", only Metadata check is running + String[] args = new String[]{"health-report", "-c", "5"}; + ScriptLauncher.handleScript(args, ScriptLauncher.getConfig(kernelImpl), testDSpaceRunnableHandler, kernelImpl); + + assertThat(testDSpaceRunnableHandler.getErrorMessages(), empty()); + List messages = testDSpaceRunnableHandler.getInfoMessages(); + + assertThat(messages, hasSize(1)); + assertThat(messages.get(0), containsString("dc.relation issues: " + " ".repeat(15) + "2")); + assertThat(messages.get(0), containsString("dc.title issues: " + " ".repeat(15) + "1")); + assertThat(messages.get(0), containsString("dc.type issues: " + " ".repeat(15) + "1")); + assertThat(messages.get(0), containsString("Error count total: " + " ".repeat(15) + "4")); + assertThat(messages.get(0), containsString("dc.subject issues: " + " ".repeat(15) + "1")); + assertThat(messages.get(0), containsString("Warning count total: " + " ".repeat(15) + "1")); + assertThat(messages.get(0), containsString("Errors:")); + assertThat(messages.get(0), containsString("Does not have dc.type metadata")); + assertThat(messages.get(0), containsString("Item has no dc.title metadata")); + assertThat(messages.get(0), containsString("does not refer back via dc.relation.isreplacedby")); + assertThat(messages.get(0), containsString("does not refer back via dc.relation.replaces")); + assertThat(messages.get(0), containsString("Warnings:")); + assertThat(messages.get(0), containsString("does not contain any [dc.subject] values")); + + ReportResultService reportResultService = ContentServiceFactory.getInstance().getReportResultService(); + List reportResults = reportResultService.findAll(context); + ReportResult reportResult = findLastReportResult(reportResults); + assertThat(reportResult.getType(), is("healthcheck")); + + JsonNode root = new ObjectMapper().readTree(reportResult.getValue()); + JsonNode metadataCheckNode = findCheckByName(root, "Metadata check"); + assertThat(metadataCheckNode, notNullValue()); + + JsonNode reportNode = metadataCheckNode.get("report"); + assertThat(reportNode, notNullValue()); + + assertThat(reportNode.get("errorCount").asInt(), is(4)); + assertThat(reportNode.get("warningCount").asInt(), is(1)); + + ArrayNode errorsNode = reportNode.withArray("errors"); + assertThat(errorsNode.size(), is(3)); + + assertThat(errorsNode.get(0).get("count").asInt(), is(2)); + assertThat(errorsNode.get(0).get("type").asText(), is("dc.relation")); + + assertThat(errorsNode.get(1).get("count").asInt(), is(1)); + assertThat(errorsNode.get(1).get("type").asText(), is("dc.title")); + + assertThat(errorsNode.get(2).get("count").asInt(), is(1)); + assertThat(errorsNode.get(2).get("type").asText(), is("dc.type")); + + ArrayNode warningsNode = reportNode.withArray("warnings"); + assertThat(warningsNode.size(), is(1)); + assertThat(warningsNode.get(0).get("count").asInt(), is(1)); + assertThat(warningsNode.get(0).get("type").asText(), is("dc.subject")); + } + + @Test + public void testMetadataCheckWithRestrictedReportSize() throws Exception { + // set max-errors-to-show to 8 and error-dispersion-quota to 1, + // This test has 14 errors in total, but the report will contain only 8 error messages. + // The errors with low frequency will be prioritized. + // The error-dispersion-quota set to 1 means that the number of errors shown + // for each error will be almost the same, in this case maximally 2 errors for each error type + context.turnOffAuthorisationSystem(); + + ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); + configurationService.setProperty("healthcheck.metadata.max-errors-to-show", 8); + configurationService.setProperty("healthcheck.metadata.error-dispersion-quota", 1); + + try { + Community community = CommunityBuilder.createCommunity(context) + .withName("Community") + .build(); + + Collection collection = CollectionBuilder.createCollection(context, community) + .withName("Collection") + .withSubmitterGroup(eperson) + .build(); + + Item item1 = ItemBuilder.createItem(context, collection) + .withTitle("Test item 1") + .withType("corpus") + .withSubject("Test subject") + .withMetadata("local", "branding", null, "Community") + .build(); + + Item item2 = ItemBuilder.createItem(context, collection) + .withTitle("Test item 2") + .withType("toolService") + .withSubject("Test subject") + .withMetadata("local", "branding", null, "Community") + .withMetadata("dc", "relation", "replaces", findItemUri(item1)) + .build(); + + ItemBuilder.createItem(context, collection) + .withTitle("Test item 3") + .withType("toolService") + .withSubject("Test subject") + .withMetadata("local", "branding", null, "Community") + .withMetadata("dc", "relation", "isreplacedby", findItemUri(item2)) + .build(); + + // create 4 items with missing title + for (int i = 0; i < 4; i++) { + ItemBuilder.createItem(context, collection) + .withType("toolService") + .withSubject("Test subject") + .withMetadata("local", "branding", null, "Community") + .build(); + } + + // create 4 items with missing type + for (int i = 4; i < 8; i++) { + ItemBuilder.createItem(context, collection) + .withTitle("Test Item " + i) + .withSubject("Test subject") + .withMetadata("local", "branding", null, "Community") + .build(); + } + + // create 4 items with duplicate type + for (int i = 8; i < 12; i++) { + ItemBuilder.createItem(context, collection) + .withTitle("Test Item " + i) + .withType("toolService") + .withType("corpus") + .withSubject("Test subject") + .withMetadata("local", "branding", null, "Community") + .build(); + } + + TestDSpaceRunnableHandler testDSpaceRunnableHandler = new TestDSpaceRunnableHandler(); + + // with "health-report -c 5", only Metadata check is running + String[] args = new String[]{"health-report", "-c", "5"}; + ScriptLauncher.handleScript(args, + ScriptLauncher.getConfig(kernelImpl), testDSpaceRunnableHandler, kernelImpl); + + assertThat(testDSpaceRunnableHandler.getErrorMessages(), empty()); + List messages = testDSpaceRunnableHandler.getInfoMessages(); + + assertThat(messages, hasSize(1)); + assertThat(messages.get(0), containsString("dc.relation issues: " + " ".repeat(15) + "2")); + assertThat(messages.get(0), containsString("dc.title issues: " + " ".repeat(15) + "4")); + assertThat(messages.get(0), containsString("dc.type issues: " + " ".repeat(15) + "4")); + assertThat(messages.get(0), containsString("duplicate value issues:" + " ".repeat(13) + "4")); + assertThat(messages.get(0), containsString("Error count total: " + " ".repeat(14) + "14")); + + assertThat(messages.get(0), containsString("Errors:")); + + // check if dc.type error is present exactly 2 times + assertThat(StringUtils.countMatches(messages.get(0), "Does not have dc.type metadata"), is(2)); + // check if dc.title error is present exactly 2 times + assertThat(StringUtils.countMatches(messages.get(0), "Item has no dc.title metadata"), is(2)); + // check if duplicate value error is present exactly 2 times + assertThat(StringUtils.countMatches(messages.get(0), "value [dc.type] is present multiple times"), is(2)); + + // check if all dc.relation errors are present + assertThat(StringUtils.countMatches( + messages.get(0), "does not refer back via dc.relation.replaces"), is(1)); + assertThat(StringUtils.countMatches( + messages.get(0), "does not refer back via dc.relation.isreplacedby"), is(1)); + assertThat(messages.get(0), containsString("and more...")); + } finally { + configurationService.setProperty("healthcheck.metadata.max-errors-to-show", null); + configurationService.setProperty("healthcheck.metadata.error-dispersion-quota", null); + } + } + + private String findItemUri(Item item) { + return ContentServiceFactory.getInstance().getItemService() + .getMetadataFirstValue(item, "dc", "identifier", "uri", Item.ANY); + } + + ReportResult findLastReportResult(List reportResults) { + return reportResults.stream().max((reportResult1, reportResult2) -> + reportResult1.getLastModified().compareTo(reportResult2.getLastModified())).orElseThrow(); + } + + JsonNode findCheckByName(JsonNode root, String checkName) { + for (JsonNode check : root.get("checks")) { + if (check.get("name").asText().equals(checkName)) { + return check; + } + } + return null; + } + } \ No newline at end of file diff --git a/dspace/config/modules/healthcheck.cfg b/dspace/config/modules/healthcheck.cfg index 972051d94ace..027e30391e26 100644 --- a/dspace/config/modules/healthcheck.cfg +++ b/dspace/config/modules/healthcheck.cfg @@ -8,7 +8,8 @@ healthcheck.checks = General Information,\ Item summary,\ User summary,\ License summary,\ - Embargo check + Embargo check,\ + Metadata check plugin.named.org.dspace.health.Check = \ org.dspace.health.InfoCheck = General Information,\ @@ -18,7 +19,8 @@ plugin.named.org.dspace.health.Check = \ org.dspace.health.ItemCheck = Item summary,\ org.dspace.health.UserCheck = User summary,\ org.dspace.health.LogAnalyserCheck = Log Analyser Check,\ - org.dspace.health.LicenseCheck = License summary + org.dspace.health.LicenseCheck = License summary,\ + org.dspace.health.MetadataCheck = Metadata check # default value of the report from the last N days (where dates are applicable) healthcheck.last_n_days = 7 From 867e43d13d08465907ae06bffd3c193c090f2561 Mon Sep 17 00:00:00 2001 From: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> Date: Mon, 29 Jun 2026 12:49:42 +0200 Subject: [PATCH 38/41] test: de-flake ItemHandleCheckerIT (mock the live handle resolver) (#1346) * test: de-flake ItemHandleCheckerIT (mock the live handle resolver) ItemHandleCheckerIT.testItemHandleNotFound intermittently failed on dtq-dev with a synthetic 617 (SocketTimeoutException) instead of 404. The checkhandles task (ItemHandleChecker) issues a HEAD request to each item's handle URL (handle.canonical.prefix + handle) with a 3s read timeout; the test pointed the prefix at the live http://hdl.handle.net/ and asserted the resolver returns 404 (non-existent handle) and 302->200 (a real handle). When the live resolver was slow/unreachable the HEAD timed out -> 617 -> red pipeline. Same class of flake as the ORCID tests; not a regression (the merge commit passed many other runs). Fix (test-only): serve the handle URLs from a local okhttp3 MockWebServer. setUp sets handle.canonical.prefix to the mock base URL; a path-based dispatcher returns 302 (+Location to a -target path) for the redirect handle, 200 for the target, and 404 otherwise. The real/invalid/ignored URLs are derived from the mock base instead of hard-coded hdl.handle.net literals; the server is closed in @After. All six tests keep their original assertions and behavior; only the live-network hop is removed. Verified locally: 6/6 green across 4 runs, no external network involved. Co-Authored-By: Claude Opus 4.8 * test: restore handle.canonical.prefix in ItemHandleCheckerIT teardown Address CodeRabbit: setUp overwrites the shared handle.canonical.prefix with the per-run mock-server URL. Capture the previous value and restore it in destroy() (after closing the mock server) so a later test in the same JVM is never left pointing at the now-closed localhost port. (The superclass destroy() reloadConfig() already resets it, but restoring explicitly makes the test self-contained.) Re-verified locally: 6/6 green. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- .../dspace/curate/ItemHandleCheckerIT.java | 75 ++++++++++++++++--- 1 file changed, 63 insertions(+), 12 deletions(-) diff --git a/dspace-api/src/test/java/org/dspace/curate/ItemHandleCheckerIT.java b/dspace-api/src/test/java/org/dspace/curate/ItemHandleCheckerIT.java index 6b1df063c106..d006c9ebccbd 100644 --- a/dspace-api/src/test/java/org/dspace/curate/ItemHandleCheckerIT.java +++ b/dspace-api/src/test/java/org/dspace/curate/ItemHandleCheckerIT.java @@ -21,6 +21,10 @@ import java.util.List; import java.util.Random; +import okhttp3.mockwebserver.Dispatcher; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; import org.dspace.AbstractIntegrationTestWithDatabase; import org.dspace.authorize.AuthorizeException; import org.dspace.builder.ItemBuilder; @@ -43,6 +47,12 @@ /** * Test for checkhandles curation task. * + *

The handle URLs are served by a local {@link MockWebServer} instead of the live handle resolver + * (http://hdl.handle.net/), which the task contacts over HTTP. Hitting the live resolver made these tests + * flaky: when the network was slow the HEAD request timed out and the task reported {@code 617} + * (SocketTimeoutException) instead of the expected status. The mock dispatcher returns deterministic + * responses keyed by path.

+ * * @author mkuchtiak */ public class ItemHandleCheckerIT extends AbstractIntegrationTestWithDatabase { @@ -55,11 +65,10 @@ public class ItemHandleCheckerIT extends AbstractIntegrationTestWithDatabase { private static final String HANDLE_ITEM3 = HANDLE_COLLECTION + "-3"; private static final String HANDLE_ITEM4 = HANDLE_COLLECTION + "-4"; private static final String HANDLE_NON_EXISTING = HANDLE_COLLECTION + "-999"; - private static final String HANDLE_URL_REAL = "http://hdl.handle.net/20.1000/5555"; - private static final String HANDLE_INVALID = HANDLE_URL_REAL + "/..??^^/"; + // Path (relative to the mock server) of a handle that the resolver answers with a 302 redirect to a 200 page. + private static final String HANDLE_REDIRECT_PATH = "20.1000/5555"; private static final String HANDLE_IGNORED_1 = "11234/998"; private static final String HANDLE_IGNORED_2 = "11234/999"; - private static final String HANDLE_URL_IGNORED = "http://hdl.handle.net/" + HANDLE_IGNORED_2; protected CommunityService communityService = ContentServiceFactory.getInstance().getCommunityService(); protected CollectionService collectionService = ContentServiceFactory.getInstance().getCollectionService(); @@ -77,15 +86,50 @@ public class ItemHandleCheckerIT extends AbstractIntegrationTestWithDatabase { private Curator curator; private CuratorReportTest.ListReporter reporter; + // Local stand-in for the handle resolver. Started in setUp(); its base URL becomes handle.canonical.prefix. + private MockWebServer mockHandleServer; + // Previous handle.canonical.prefix, captured in setUp and restored in destroy so the shared config is not + // left pointing at the now-closed mock server for later tests in the same JVM. + private String originalHandlePrefix; + // URLs that point at the mock server (computed from its dynamic port in setUp). + private String handleUrlReal; + private String handleUrlRedirectTarget; + private String handleInvalid; + private String handleUrlIgnored; + @Before @Override public void setUp() throws Exception { super.setUp(); CoreServiceFactory.getInstance().getPluginService().clearNamedPluginClasses(); try { + // Serve handle URLs from a local mock server so the task never contacts the live resolver. + mockHandleServer = new MockWebServer(); + String baseUrl = mockHandleServer.url("/").toString(); + handleUrlReal = baseUrl + HANDLE_REDIRECT_PATH; + handleUrlRedirectTarget = baseUrl + HANDLE_REDIRECT_PATH + "-target"; + handleInvalid = handleUrlReal + "/..??^^/"; + handleUrlIgnored = baseUrl + HANDLE_IGNORED_2; + mockHandleServer.setDispatcher(new Dispatcher() { + @Override + public MockResponse dispatch(RecordedRequest request) { + String path = request.getPath(); + if (("/" + HANDLE_REDIRECT_PATH).equals(path)) { + // a "real" handle: 302 redirect; the task follows redirects manually via the Location header + return new MockResponse().setResponseCode(302).setHeader("Location", handleUrlRedirectTarget); + } + if (("/" + HANDLE_REDIRECT_PATH + "-target").equals(path)) { + return new MockResponse().setResponseCode(200); + } + // any other (well-formed, non-ignored) handle URL is treated as "not found" + return new MockResponse().setResponseCode(404); + } + }); + //we have to create a new community in the database context.turnOffAuthorisationSystem(); - cfg.setProperty("handle.canonical.prefix", "http://hdl.handle.net/"); + originalHandlePrefix = cfg.getProperty("handle.canonical.prefix"); + cfg.setProperty("handle.canonical.prefix", baseUrl); cfg.setProperty("curate.checklist.ignore", HANDLE_IGNORED_1 + "," + HANDLE_IGNORED_2); this.parentCommunity = communityService.create(null, context); @@ -131,7 +175,7 @@ public void testItemHandleNotFound() throws IOException { @Test public void testItemHandleRedirected() throws IOException { - replaceHandleUrl(item2, HANDLE_URL_REAL); + replaceHandleUrl(item2, handleUrlReal); curator.curate(context, HANDLE_ITEM2); assertEquals("Curation should succeed", Curator.CURATE_SUCCESS, curator.getStatus(TASK_NAME)); assertTrue(curator.getResult(TASK_NAME).contains(redirectedResultForItem(item2))); @@ -150,18 +194,18 @@ public void testNonExistingHandle() throws IOException { @Test public void testInvalidHandleUrl() throws IOException { - replaceHandleUrl(item3, HANDLE_INVALID); + replaceHandleUrl(item3, handleInvalid); curator.curate(context, HANDLE_ITEM3); assertEquals("Curation should fail", Curator.CURATE_FAIL, curator.getStatus(TASK_NAME)); String singleReport = reporter.getReport().get(0); - assertTrue(singleReport.contains(HANDLE_INVALID + " = 500 - FAILED\n")); + assertTrue(singleReport.contains(handleInvalid + " = 500 - FAILED\n")); assertTrue(singleReport.contains("Error: java.net.URISyntaxException: Illegal character")); reporter.getReport().clear(); } @Test public void testHandleUrlIgnored() throws IOException { - replaceHandleUrl(item4, HANDLE_URL_IGNORED); + replaceHandleUrl(item4, handleUrlIgnored); curator.curate(context, HANDLE_ITEM4); assertEquals("Curation should skip", Curator.CURATE_SKIP, curator.getStatus(TASK_NAME)); assertEquals("Item: " + HANDLE_ITEM4 + "\n", reporter.getReport().get(0)); @@ -170,9 +214,9 @@ public void testHandleUrlIgnored() throws IOException { @Test public void testCurateCollection() throws IOException { - replaceHandleUrl(item2, HANDLE_URL_REAL); - replaceHandleUrl(item3, HANDLE_INVALID); - replaceHandleUrl(item4, HANDLE_URL_IGNORED); + replaceHandleUrl(item2, handleUrlReal); + replaceHandleUrl(item3, handleInvalid); + replaceHandleUrl(item4, handleUrlIgnored); curator.curate(context, HANDLE_COLLECTION); // the final curator status is derived from the status of the latest checked item // so the final curator status is unpredictable @@ -184,7 +228,7 @@ public void testCurateCollection() throws IOException { both( containsString("Item: " + item2.getHandle())).and(containsString(" = 200 - OK\n") ), // item2 - containsString(HANDLE_INVALID + " = 500 - FAILED"), // item 3 + containsString(handleInvalid + " = 500 - FAILED"), // item 3 is("Item: " + HANDLE_ITEM4 + "\n") // item 4 (ignored) )); } @@ -192,6 +236,13 @@ public void testCurateCollection() throws IOException { @After @Override public void destroy() throws Exception { + if (mockHandleServer != null) { + mockHandleServer.close(); + } + // restore the shared config so a later test is not left pointing at the now-closed mock server + if (originalHandlePrefix != null) { + cfg.setProperty("handle.canonical.prefix", originalHandlePrefix); + } // remove all registered handles properly identifierService.delete(context, item1, HANDLE_ITEM1); identifierService.delete(context, item2, HANDLE_ITEM2); From 74f5862748412bebb6bb7ab43952308871779bf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Wed, 1 Jul 2026 16:43:28 +0200 Subject: [PATCH 39/41] UFAL/Obtain special groups from user context when new token is generated (on token refresh) (ufal/clarin-dspace#1378) (#1347) * Issue 1373: obtain special groups from user context when new token is generated (on token refresh) * resolve Copilot comments * resolve Copilot Comments: compute special groups only when when user is authenticated * Remove HttpSession dependency from ClarinShibAuthentication Use request-scoped attributes for shib.authenticated instead of HttpSession/JSESSIONID, aligning with upstream ShibAuthentication. Follow-up to ufal/clarin-dspace#1373/ufal/clarin-dspace#1378. * Guard against null special groups in Context.getSpecialGroups A special-group UUID may reference a Group that has since been deleted; GroupService.find returns null in that case. The list was built with an unconditional add, so it could contain null elements, which caused an NPE downstream (e.g. SpecialGroupClaimProvider.getValue maps group.getID() while generating the JWT sg claim on token refresh). Filter nulls once here so every caller is covered. Follow-up to ufal/clarin-dspace#1373/ufal/clarin-dspace#1378. --------- (cherry picked from commit 4c294b24585a8581361aeb59aa2fb160ca9801c6) Co-authored-by: Milan Kuchtiak --- .../clarin/ClarinShibAuthentication.java | 41 ++++++++----------- .../main/java/org/dspace/core/Context.java | 7 +++- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/authenticate/clarin/ClarinShibAuthentication.java b/dspace-api/src/main/java/org/dspace/authenticate/clarin/ClarinShibAuthentication.java index ba5d8cd65bfa..ef0b9e4b0a82 100644 --- a/dspace-api/src/main/java/org/dspace/authenticate/clarin/ClarinShibAuthentication.java +++ b/dspace-api/src/main/java/org/dspace/authenticate/clarin/ClarinShibAuthentication.java @@ -277,7 +277,7 @@ public int authenticate(Context context, String username, String password, // Step 4: Log the user in. context.setCurrentUser(eperson); - request.getSession().setAttribute("shib.authenticated", true); + request.setAttribute("shib.authenticated", true); AuthenticateServiceFactory.getInstance().getAuthenticationService().initEPerson(context, request, eperson); log.info(eperson.getEmail() + " has been authenticated via shibboleth."); @@ -330,42 +330,35 @@ public int authenticate(Context context, String username, String password, @Override public List getSpecialGroups(Context context, HttpServletRequest request) { try { - // User has not successfuly authenticated via shibboleth. - if (request == null || - context.getCurrentUser() == null || - request.getSession().getAttribute("shib.authenticated") == null) { - return Collections.EMPTY_LIST; + // User has not successfully authenticated via shibboleth. + if (request == null || context.getCurrentUser() == null) { + return Collections.emptyList(); } - // If we have already calculated the special groups then return them. - if (request.getSession().getAttribute("shib.specialgroup") != null) { - log.debug("Returning cached special groups."); - List sessionGroupIds = (List) request.getSession().getAttribute("shib.specialgroup"); - List result = new ArrayList<>(); - for (UUID uuid : sessionGroupIds) { - result.add(groupService.find(context, uuid)); - } - return result; + List specialGroups = context.getSpecialGroups(); + if (!specialGroups.isEmpty()) { + log.debug("Returning special groups from context."); + return specialGroups; } + if (request.getAttribute("shib.authenticated") == null) { + log.debug("User has not been authenticated via shibboleth, returning empty list of special groups."); + return Collections.emptyList(); + } List groupIds = new ShibGroup(new ShibHeaders(request), context).get(); - // Cache the special groups, so we don't have to recalculate them again - // for this session. - request.getSession().setAttribute("shib.specialgroup", groupIds); List groups = new ArrayList<>(); for (UUID uuid : groupIds) { Group foundGroup = groupService.find(context, uuid); - if (Objects.isNull(foundGroup)) { - continue; + if (foundGroup != null) { + groups.add(foundGroup); } - groups.add(foundGroup); } return groups; } catch (Throwable t) { - log.error("Unable to validate any sepcial groups this user may belong too because of an exception.", t); - return Collections.EMPTY_LIST; + log.error("Unable to validate any special groups this user may belong to because of an exception.", t); + return Collections.emptyList(); } } @@ -1291,7 +1284,7 @@ private String getShibURL(HttpServletRequest request) { public boolean isUsed(final Context context, final HttpServletRequest request) { if (request != null && context.getCurrentUser() != null && - request.getSession().getAttribute("shib.authenticated") != null) { + request.getAttribute("shib.authenticated") != null) { return true; } return false; diff --git a/dspace-api/src/main/java/org/dspace/core/Context.java b/dspace-api/src/main/java/org/dspace/core/Context.java index e721deff5e71..3f615d4ebfb2 100644 --- a/dspace-api/src/main/java/org/dspace/core/Context.java +++ b/dspace-api/src/main/java/org/dspace/core/Context.java @@ -714,7 +714,12 @@ public boolean inSpecialGroup(UUID groupID) { public List getSpecialGroups() throws SQLException { List myGroups = new ArrayList<>(); for (UUID groupId : specialGroups) { - myGroups.add(EPersonServiceFactory.getInstance().getGroupService().find(this, groupId)); + Group group = EPersonServiceFactory.getInstance().getGroupService().find(this, groupId); + // A special group UUID may reference a group that has since been deleted; skip nulls + // so callers never receive a list containing null (avoids NPE downstream). + if (group != null) { + myGroups.add(group); + } } return myGroups; From e9392ae1917c08b2d04f06c8f6ad0428a8ac74b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Wed, 1 Jul 2026 16:49:33 +0200 Subject: [PATCH 40/41] UFAL/fix: DOI Organizer creates duplicate dc.identifier.doi metadata (ufal/clarin-dspace#1368) (#1350) * Issue 1361 fix: DOI Organizer creates duplicate dc.identifier.doi metadata * copilot comments * Issue 1361: make DOI metadata save additive, report duplicates via QA saveDOIToObject now adds dc.identifier.doi only when that exact value is not already present, and no longer deletes metadata. The previous fix cleared existing values when more than one was found or when a different DOI was present; since this method runs after the DOI has already been registered with the external agency, silently dropping a (possibly legacy/citable) identifier is lossy and irreversible. The operation stays idempotent, so re-registration no longer creates duplicate values. Items that legitimately end up with more than one dc.identifier.doi value are now surfaced for manual review by adding dc.identifier.doi to the ItemMetadataQAChecker noDuplicate list (metadataqa curation task) rather than being cleaned up silently in the write path. Tests: flip the replace test to assert a pre-existing different DOI is preserved alongside the new one, keep the idempotency test, and add a QA checker IT asserting an item with two DOIs fails curation. * local field renaming --------- (cherry picked from commit 3b7db4ca38c5bd7b047cb2861d8ee23fbec215e4) Co-authored-by: Milan Kuchtiak --- .../ctask/general/ItemMetadataQAChecker.java | 1 + .../identifier/DOIIdentifierProvider.java | 25 ++++-- .../curate/ItemMetadataQACheckerIT.java | 22 ++++++ .../identifier/DOIIdentifierProviderTest.java | 76 +++++++++++++++++++ 4 files changed, 118 insertions(+), 6 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/ctask/general/ItemMetadataQAChecker.java b/dspace-api/src/main/java/org/dspace/ctask/general/ItemMetadataQAChecker.java index c4e5ef709b67..2f813b01f232 100644 --- a/dspace-api/src/main/java/org/dspace/ctask/general/ItemMetadataQAChecker.java +++ b/dspace-api/src/main/java/org/dspace/ctask/general/ItemMetadataQAChecker.java @@ -87,6 +87,7 @@ public void init(Curator curator, String taskId) throws IOException { "dc.rights.label", "dc.date.available", "dc.source.uri", + "dc.identifier.doi", "metashare.ResourceInfo#DistributionInfo#LicenseInfo.license" }); strangeMetadata = configurationService.getArrayProperty("lr.curation.metadata.strange", new String[]{ diff --git a/dspace-api/src/main/java/org/dspace/identifier/DOIIdentifierProvider.java b/dspace-api/src/main/java/org/dspace/identifier/DOIIdentifierProvider.java index dd48131b1fdf..ff5097aa5e5d 100644 --- a/dspace-api/src/main/java/org/dspace/identifier/DOIIdentifierProvider.java +++ b/dspace-api/src/main/java/org/dspace/identifier/DOIIdentifierProvider.java @@ -1067,13 +1067,26 @@ protected void saveDOIToObject(Context context, DSpaceObject dso, String doi) } Item item = (Item) dso; - itemService.addMetadata(context, item, MD_SCHEMA, DOI_ELEMENT, DOI_QUALIFIER, null, - doiService.DOIToExternalForm(doi)); - try { - itemService.update(context, item); - } catch (SQLException | AuthorizeException ex) { - throw ex; + String doiURL = doiService.DOIToExternalForm(doi); + + // Add the DOI to the metadata only if this exact value is not present yet. This keeps the operation + // idempotent (re-registration does not create duplicate values) without ever deleting metadata: a + // pre-existing, different DOI is left untouched. This method is called after the DOI has already been + // registered with the external agency, so destroying metadata here would be lossy and irreversible. + // Items that end up with more than one dc.identifier.doi value are surfaced by the ItemMetadataQAChecker + // curation task for manual review. + List existing = itemService.getMetadata(item, MD_SCHEMA, DOI_ELEMENT, DOI_QUALIFIER, Item.ANY); + boolean alreadyPresent = existing.stream() + .anyMatch(metadataValue -> doiURL.equals(metadataValue.getValue())); + + if (alreadyPresent) { + log.debug("The DOI {} is already part of the metadata of Item {}. Not adding it again.", + doi, item.getID()); + return; } + + itemService.addMetadata(context, item, MD_SCHEMA, DOI_ELEMENT, DOI_QUALIFIER, null, doiURL); + itemService.update(context, item); } /** diff --git a/dspace-api/src/test/java/org/dspace/curate/ItemMetadataQACheckerIT.java b/dspace-api/src/test/java/org/dspace/curate/ItemMetadataQACheckerIT.java index 315a04a68b11..446e5e809daf 100644 --- a/dspace-api/src/test/java/org/dspace/curate/ItemMetadataQACheckerIT.java +++ b/dspace-api/src/test/java/org/dspace/curate/ItemMetadataQACheckerIT.java @@ -63,6 +63,7 @@ public class ItemMetadataQACheckerIT extends AbstractIntegrationTestWithDatabase Item itemWithIncorrectLanguageName; Item itemWithTwoAvailableDates; Item itemWithTwoAvailableDatesAndLang; + Item itemWithTwoDois; Item itemVersion1; Item itemVersion2; Item itemVersion3; @@ -146,6 +147,13 @@ public void setUp() throws Exception { itemService.addMetadata(context, itemWithTwoAvailableDatesAndLang,"dc", "date", "available", "en_US", "2021-01-01"); + itemWithTwoDois = ItemBuilder.createItem(context, collection) + .withTitle("Item With Two DOIs") + .withMetadata("dc", "type", null, "corpus") + .withMetadata("dc", "identifier", "doi", "https://doi.org/10.5072/test-1") + .withMetadata("dc", "identifier", "doi", "https://doi.org/10.5072/test-2") + .build(); + itemVersion1 = ItemBuilder.createItem(context, collection) .withTitle("Item Version 1") .withMetadata("dc", "type", null, "corpus") @@ -233,6 +241,20 @@ public void testItemWithTwoAvailableDatesAndLang() throws IOException { assertTrue("Result should mention multiple dc.date.available", result.contains("dc.date.available")); } + @Test + public void testItemWithTwoDois() throws IOException { + Curator curator = new Curator(); + curator.addTask(TASK_NAME); + context.setCurrentUser(admin); + + // Run curator task for item with two dc.identifier.doi - should fail + curator.curate(context, itemWithTwoDois.getHandle()); + int status = curator.getStatus(TASK_NAME); + assertEquals("Curation should fail for item with two dc.identifier.doi", Curator.CURATE_FAIL, status); + String result = curator.getResult(TASK_NAME); + assertTrue("Result should mention multiple dc.identifier.doi", result.contains("dc.identifier.doi")); + } + @Test public void testValidItem() throws IOException { Curator curator = new Curator(); diff --git a/dspace-api/src/test/java/org/dspace/identifier/DOIIdentifierProviderTest.java b/dspace-api/src/test/java/org/dspace/identifier/DOIIdentifierProviderTest.java index 43f93e606dfd..7ef10de02311 100644 --- a/dspace-api/src/test/java/org/dspace/identifier/DOIIdentifierProviderTest.java +++ b/dspace-api/src/test/java/org/dspace/identifier/DOIIdentifierProviderTest.java @@ -23,6 +23,7 @@ import java.util.Date; import java.util.List; import java.util.Random; +import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; @@ -332,6 +333,51 @@ public void testStore_DOI_as_item_metadata() assertTrue("Cannot store DOI as item metadata value.", result); } + @Test + public void testStore_DOI_keeps_existing_different_doi_metadata() throws SQLException, AuthorizeException, + IOException, IdentifierException, IllegalAccessException, WorkflowException { + Item item = newItem(); + + // this checks that the method does not fail if there is already a *different* DOI in the metadata, + // here we verify that the existing DOI is preserved (not deleted) and the new one is added alongside it. + // Items with more than one DOI are reported by the ItemMetadataQAChecker curation task, not silently + // cleaned up here. + String oldDoi = DOI.SCHEME + PREFIX + "/" + NAMESPACE_SEPARATOR + "1234"; + String newDoi = DOI.SCHEME + PREFIX + "/" + NAMESPACE_SEPARATOR + Long.toHexString(new Date().getTime()); + + context.turnOffAuthorisationSystem(); + itemService.addMetadata(context, item, DOIIdentifierProvider.MD_SCHEMA, + DOIIdentifierProvider.DOI_ELEMENT, + DOIIdentifierProvider.DOI_QUALIFIER, + null, + doiService.DOIToExternalForm(oldDoi)); + provider.saveDOIToObject(context, item, newDoi); + context.restoreAuthSystemState(); + + checkDoiMetadata(item, oldDoi, newDoi); + } + + @Test + public void testStore_DOI_check_single_doi_metadata() throws SQLException, AuthorizeException, IOException, + IdentifierException, IllegalAccessException, WorkflowException { + Item item = newItem(); + + // this checks that the method does not fail if there is already a DOI in the metadata, + // here we check if DOI metadata are not duplicated + String doi = DOI.SCHEME + PREFIX + "/" + NAMESPACE_SEPARATOR + Long.toHexString(new Date().getTime()); + + context.turnOffAuthorisationSystem(); + itemService.addMetadata(context, item, DOIIdentifierProvider.MD_SCHEMA, + DOIIdentifierProvider.DOI_ELEMENT, + DOIIdentifierProvider.DOI_QUALIFIER, + null, + doiService.DOIToExternalForm(doi)); + provider.saveDOIToObject(context, item, doi); + context.restoreAuthSystemState(); + + checkSingleDoiMetadata(item, doi); + } + @Test public void testGet_DOI_out_of_item_metadata() throws SQLException, AuthorizeException, IOException, IdentifierException, IllegalAccessException, @@ -868,4 +914,34 @@ public void testLoadOrCreateDOIReturnsMintedStatus() // registerOnline // reserveOnline + private void checkSingleDoiMetadata(Item item, String doi) throws IdentifierException { + List metadata = itemService.getMetadata(item, DOIIdentifierProvider.MD_SCHEMA, + DOIIdentifierProvider.DOI_ELEMENT, + DOIIdentifierProvider.DOI_QUALIFIER, + Item.ANY); + boolean result = false; + if (metadata.size() == 1 && metadata.get(0).getValue().equals(doiService.DOIToExternalForm(doi))) { + result = true; + } + assertTrue("Invalid or duplicate 'dc.identifier.doi' metadata value(s).", result); + } + + private void checkDoiMetadata(Item item, String... dois) throws IdentifierException { + List values = itemService.getMetadata(item, DOIIdentifierProvider.MD_SCHEMA, + DOIIdentifierProvider.DOI_ELEMENT, + DOIIdentifierProvider.DOI_QUALIFIER, + Item.ANY) + .stream() + .map(MetadataValue::getValue) + .collect(Collectors.toList()); + + List expected = new ArrayList<>(); + for (String doi : dois) { + expected.add(doiService.DOIToExternalForm(doi)); + } + + assertEquals("Unexpected number of 'dc.identifier.doi' metadata values.", expected.size(), values.size()); + assertTrue("Expected 'dc.identifier.doi' metadata values are missing.", values.containsAll(expected)); + } + } From 00501a2db085717984aee5c8e6c0a48372185bb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Mon, 13 Jul 2026 12:42:31 +0200 Subject: [PATCH 41/41] [Port to dtq-dev] Issue ufal/clarin-dspace#1351 simple ror authority (#1349) * Issue ufal/clarin-dspace#1351 simple ror authority (ufal/clarin-dspace#1352) * Issue 1351: SimpleRORAuthority * integration test * code cleanup * add debug messages to find test failure * Revert "add debug messages to find test failure" This reverts commit 286fde46a39830cb45ba932fd00702b4c2c2bfa2. * test failures * Revert "test failures" This reverts commit 8808f5eead39720219f4a4c9e641c22f4f75052c. * resolve Copilot comments * resolve PR comments, add ROR lookup to Publisher field * rollback changes in VocabularyEntryLinkRepository * fixing failing tests * implement PR Comments * implementation improvement * resolve PR comments (O.Kosarko) * Address review nits: shared ObjectMapper, commons-lang3, IT cleanup - Reuse a single static ObjectMapper in SimpleRORAuthority instead of constructing one per call. - Switch to the non-deprecated org.apache.commons.lang3.LocaleUtils. - Reset the ChoiceAuthority plugin configuration in @AfterClass so VocabularyEntryLinkRepositoryIT no longer leaks the SimpleRORAuthority registration into other integration tests. * implement cache for gertLabel() * resolve PR Comments --------- Co-authored-by: Ondrej Kosarko (cherry picked from commit 25a5a25f023d9d4d9280a7b4bf7b8c7b32eca144) * fix potential NPE Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * fix possible NPE in getBestMatch * fix possible NPE and some typos * add license header * Don't cache the fallback * resolved CodeRabbit comment --------- Co-authored-by: Milan Kuchtiak Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../content/authority/SimpleRORAuthority.java | 119 + .../org/dspace/external/RorRestConnector.java | 350 +++ .../dspace/external/model/ror/Location.java | 79 + .../dspace/external/model/ror/RorItem.java | 92 + .../dspace/external/model/ror/RorItems.java | 50 + .../org/dspace/external/ror/CacheLogger.java | 27 + .../spring/api/ror-authority-services.xml | 15 + .../dspace/external/MockRorRestConnector.java | 69 + .../dspace/external/ror/UniversityOfPisa.json | 2172 +++++++++++++++++ .../external/ror/UniversityOfPisaByID.json | 128 + .../ror/UniversityOfPisaByQueryExact.json | 169 ++ .../rest/VocabularyEntryLinkRepositoryIT.java | 193 ++ dspace/config/ehcache.xml | 22 + dspace/config/features/enable-ror.cfg | 31 + .../spring/api/ror-authority-services.xml | 23 + 15 files changed, 3539 insertions(+) create mode 100644 dspace-api/src/main/java/org/dspace/content/authority/SimpleRORAuthority.java create mode 100644 dspace-api/src/main/java/org/dspace/external/RorRestConnector.java create mode 100644 dspace-api/src/main/java/org/dspace/external/model/ror/Location.java create mode 100644 dspace-api/src/main/java/org/dspace/external/model/ror/RorItem.java create mode 100644 dspace-api/src/main/java/org/dspace/external/model/ror/RorItems.java create mode 100644 dspace-api/src/main/java/org/dspace/external/ror/CacheLogger.java create mode 100644 dspace-api/src/test/data/dspaceFolder/config/spring/api/ror-authority-services.xml create mode 100644 dspace-api/src/test/java/org/dspace/external/MockRorRestConnector.java create mode 100644 dspace-api/src/test/resources/org/dspace/external/ror/UniversityOfPisa.json create mode 100644 dspace-api/src/test/resources/org/dspace/external/ror/UniversityOfPisaByID.json create mode 100644 dspace-api/src/test/resources/org/dspace/external/ror/UniversityOfPisaByQueryExact.json create mode 100644 dspace-server-webapp/src/test/java/org/dspace/app/rest/VocabularyEntryLinkRepositoryIT.java create mode 100644 dspace/config/features/enable-ror.cfg create mode 100644 dspace/config/spring/api/ror-authority-services.xml diff --git a/dspace-api/src/main/java/org/dspace/content/authority/SimpleRORAuthority.java b/dspace-api/src/main/java/org/dspace/content/authority/SimpleRORAuthority.java new file mode 100644 index 000000000000..80425bd977c4 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/content/authority/SimpleRORAuthority.java @@ -0,0 +1,119 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content.authority; + +import org.dspace.external.RorRestConnector; +import org.dspace.utils.DSpace; + +/** + * ChoiceAuthority using the ROR API. + * + * @author Milan Kuchtiak + */ +public class SimpleRORAuthority implements ChoiceAuthority { + + private String pluginInstanceName; + + private final RorRestConnector rorRestConnector = new DSpace().getServiceManager().getServiceByName( + "RorRestConnector", RorRestConnector.class); + + /** + * Get all values from the authority that match the preferred value. + * Note that the offering was entered by the user and may contain + * mixed/incorrect case, whitespace, etc so the plugin should be careful + * to clean up user data before making comparisons. + *

+ * Value of a "Name" field will be in canonical DSpace person name format, + * which is "Lastname, Firstname(s)", e.g. "Smith, John Q.". + *

+ * Some authorities with a small set of values may simply return the whole + * set for any sample value, although it's a good idea to set the + * defaultSelected index in the Choices instance to the choice, if any, + * that matches the value. + * + * @param text user's value to match + * @param start choice at which to start, 0 is first. + * @param limit maximum number of choices to return, 0 for no limit. + * @param locale explicit localization key if available, or null + * @return a Choices object (never null). + */ + @Override + public Choices getMatches(String text, int start, int limit, String locale) { + return rorRestConnector.getMatches(text, start, limit, locale); + } + + /** + * Get the single "best" match (if any) of a value in the authority + * to the given user value. The "confidence" element of Choices is + * expected to be set to a meaningful value about the circumstances of + * this match. + *

+ * This call is typically used in non-interactive metadata ingest + * where there is no interactive agent to choose from among options. + * + * @param text user's value to match + * @param locale explicit localization key if available, or null + * @return a Choices object (never null) with 1 or 0 values. + */ + @Override + public Choices getBestMatch(String text, String locale) { + return rorRestConnector.getBestMatch(text, locale); + } + + @Override + public Choice getChoice(String authKey, String locale) { + return rorRestConnector.getChoice(authKey, locale); + } + + /** + * Get the canonical user-visible "label" (i.e. short descriptive text) + * for a key in the authority. Can be localized given the implicit + * or explicit locale specification. + *

+ * This may get called many times while populating a Web page so it should + * be implemented as efficiently as possible. + * + * @param key authority key known to this authority. + * @param locale explicit localization key if available, or null + * @return descriptive label - should always return something, never null. + */ + @Override + public String getLabel(String key, String locale) { + return rorRestConnector.getLabel(key, locale); + } + + /** + * Get the instance's particular name. + * Returns the name by which the class was chosen when + * this instance was created. Only works for instances created + * by PluginService, or if someone remembers to call setPluginName. + *

+ * Useful when the implementation class wants to be configured differently + * when it is invoked under different names. + * + * @return name or null if not available. + */ + @Override + public String getPluginInstanceName() { + return pluginInstanceName; + } + + /** + * Set the name under which this plugin was instantiated. + * Not to be invoked by application code, it is + * called automatically by PluginService.getNamedPlugin() + * when the plugin is instantiated. + * + * @param name -- name used to select this class. + */ + @Override + public void setPluginInstanceName(String name) { + this.pluginInstanceName = name; + } + +} diff --git a/dspace-api/src/main/java/org/dspace/external/RorRestConnector.java b/dspace-api/src/main/java/org/dspace/external/RorRestConnector.java new file mode 100644 index 000000000000..fb36dfa08349 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/external/RorRestConnector.java @@ -0,0 +1,350 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.external; + +import java.io.InputStream; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.stream.Collectors; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.core.Response; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.LocaleUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.dspace.content.authority.Choice; +import org.dspace.content.authority.Choices; +import org.dspace.external.model.ror.Location; +import org.dspace.external.model.ror.RorItem; +import org.dspace.external.model.ror.RorItems; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; +import org.springframework.cache.annotation.Cacheable; + +/** + * REST connector for ROR API. It is used by RORAuthority to retrieve data from ROR API. + * + * @author Milan Kuchtiak + */ +public class RorRestConnector { + + private static final Logger log = LogManager.getLogger(RorRestConnector.class); + + static final String ROR_ID_PATTERN = "^0[a-z0-9]{6}[0-9]{2}$"; + + // this is the number of items returned by the ROR API in each page + private static final int ROR_ITEMS_COUNT = 20; + // maximum number of pages that can be returned by the ROR API is 500 + private static final int ROR_MAX_PAGES = 500; + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static final Client client = ClientBuilder.newClient(); + + private String apiUrl; + private String clientId; + + public void setApiUrl(String apiUrl) { + this.apiUrl = apiUrl; + } + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + public Response getByQuery(String query) { + return getByQuery(query, 1); + } + + public Response getByQuery(String query, int page) { + return client.target(apiUrl) + .queryParam("query", query) + .queryParam("page", page) + .request() + .header("Client-Id", clientId) + .accept("application/json") + .get(); + } + + public Response getByID(String rorID) { + if (rorID != null && rorID.matches(ROR_ID_PATTERN)) { + return client.target(apiUrl).path(rorID) + .request() + .header("Client-Id", clientId) + .accept("application/json") + .get(); + } else { + return Response.status(Response.Status.NOT_FOUND).build(); + } + } + + @Cacheable(cacheNames = "ror-labels", key = "#rorID + '_' + #locale", + unless = "#result == null || #result.equals(#rorID)") + public String getLabel(String rorID, String locale) { + Choice choice = getChoice(rorID, locale); + return choice != null ? choice.label : rorID; + } + + public Choices getMatches(String text, int start, int limit, String locale) { + if (text == null || text.trim().isEmpty()) { + return new Choices(true); + } + + // allow only limits that are a divisor of ROR_RESULTS_COUNT(20), + // to avoid pagination complication in the UI + if (limit <= 0) { + limit = ROR_ITEMS_COUNT; + } else if (limit > ROR_ITEMS_COUNT || ROR_ITEMS_COUNT % limit != 0) { + throw new IllegalArgumentException("The page size must be a divisor of " + ROR_ITEMS_COUNT + "."); + } + + // calculate the offset (page parameter) to use in the ROR API call + int offset = start / ROR_ITEMS_COUNT; + + // if the offset is too high, it means the user is trying to access a page that doesn't exist, + // so we return an empty result instead of making an API call + if (offset + 1 > ROR_MAX_PAGES) { + throw new IllegalArgumentException("Exceeded maximal page number for the ROR API, which is " + + (ROR_MAX_PAGES * (ROR_ITEMS_COUNT / limit) - 1) + ", for page size " + limit + "."); + } + + try (Response response = getByQuery(text, offset + 1)) { + if (response.getStatus() == Response.Status.OK.getStatusCode()) { + try (InputStream is = response.readEntity(InputStream.class)) { + RorItems rorItems = OBJECT_MAPPER.readValue(is, RorItems.class); + int total = rorItems.getNoOfResults(); + List items = rorItems.getItems(); + if (items.isEmpty()) { + return new Choices(new Choice[0], start, total, Choices.CF_NOTFOUND, false); + } + + String localeLanguage = getLocaleLanguage(locale); + + StoredNameType storedNameType = resolveStoredNameType(); + List choices = items.stream() + .map(item -> toChoice(item, localeLanguage, storedNameType)) + .collect(Collectors.toList()); + + // select sublist of results to return based on the start and limit parameters + int startIndex = 0; + if (limit != ROR_ITEMS_COUNT) { + startIndex = start % ROR_ITEMS_COUNT; + if (startIndex >= choices.size()) { + // the start index is greater than the choices size + // so we cannot select a sublist of results + return new Choices(new Choice[0], start, total, Choices.CF_NOTFOUND, false); + } + int endIndex = Math.min(startIndex + limit, choices.size()); + choices = choices.subList(startIndex, endIndex); + } + + int confidence = choices.isEmpty() ? Choices.CF_NOTFOUND : + choices.size() == 1 ? Choices.CF_UNCERTAIN : Choices.CF_AMBIGUOUS; + + return new Choices(choices.toArray(Choice[]::new), start, total, + confidence, total > (offset * ROR_ITEMS_COUNT + startIndex + choices.size())); + } catch (Exception e) { + log.error("Error during search", e); + } + } + } + return new Choices(true); + } + + public Choices getBestMatch(String text, String locale) { + if (text == null || text.trim().isEmpty()) { + return new Choices(true); + } + try (Response response = getByQuery(sanitizeQuery(text))) { + if (response.getStatus() == Response.Status.OK.getStatusCode()) { + try (InputStream is = response.readEntity(InputStream.class)) { + RorItems rorItems = OBJECT_MAPPER.readValue(is, RorItems.class); + List items = rorItems.getItems(); + if (items.isEmpty()) { + return new Choices(false); + } + Choice[] choices = {toChoice(items.get(0), getLocaleLanguage(locale), resolveStoredNameType())}; + return new Choices(choices, 0, 1, Choices.CF_UNCERTAIN, false); + } catch (Exception e) { + log.error("Error during search", e); + } + } + } + + return new Choices(true); + } + + public Choice getChoice(String authKey, String locale) { + try (Response response = getByID(authKey)) { + if (response.getStatus() == Response.Status.OK.getStatusCode()) { + try (InputStream is = response.readEntity(InputStream.class)) { + RorItem rorItem = OBJECT_MAPPER.readValue(is, RorItem.class); + return RorRestConnector.toChoice(rorItem, getLocaleLanguage(locale), resolveStoredNameType()); + } catch (Exception e) { + log.error("Error during search", e); + } + } + } + return null; + } + + private static String getLocaleLanguage(String locale) { + try { + return Optional.ofNullable(LocaleUtils.toLocale(locale)).map(Locale::getLanguage).orElse("en"); + } catch (IllegalArgumentException e) { + log.warn("Invalid locale format: " + locale + ", using default 'en' locale."); + return "en"; + } + } + + private static Choice toChoice(RorItem rorItem, String localeLanguage, StoredNameType storedNameType) { + String authority = rorItem.getId(); + int slashIndex = authority.lastIndexOf("/"); + if (slashIndex != -1) { + authority = authority.substring(slashIndex + 1); + } + + Choice c = new Choice(); + c.authority = authority; + + List names = rorItem.getNames(); + if (!names.isEmpty()) { + String label = null; + String rorDisplay = null; + String enLabel = null; + StringBuilder aliases = new StringBuilder(); + // the label quality is the following: + // 4 - locale label from labels, 3 - locale label from aliases, 2 - english label, 1 - any other label + int labelQuality = 0; + // the enLabelQuality is the following: + // 2 - english label from labels, 1 - english label from aliases + int enLabelQuality = 0; + + for (RorItem.Name name : names) { + if (rorDisplay == null && name.getTypes().contains("ror_display")) { + rorDisplay = name.getValue(); + } + if (name.getTypes().contains("label")) { + if (enLabelQuality < 2 && "en".equals(name.getLang())) { + enLabelQuality = 2; + enLabel = name.getValue(); + } + if (labelQuality < 4 && localeLanguage.equals(name.getLang())) { + labelQuality = 4; + label = name.getValue(); + } else if (labelQuality < 2 && "en".equals(name.getLang())) { + labelQuality = 2; + label = name.getValue(); + } else if (labelQuality < 1) { + labelQuality = 1; + label = name.getValue(); + } + } + + if (name.getTypes().contains("alias")) { + String lang = name.getLang(); + if (enLabelQuality < 1 && "en".equals(lang)) { + enLabelQuality = 1; + enLabel = name.getValue(); + } + if (labelQuality < 3 && localeLanguage.equals(lang)) { + labelQuality = 3; + label = name.getValue(); + } + if (aliases.length() > 0) { + aliases.append(", "); + } + aliases.append(name.getValue()); + if (lang != null) { + aliases.append(" (").append(lang).append(")"); + } + } + } + + // fallback for label value if there is no label with type "label" in the ROR response + if (label == null) { + label = (rorDisplay != null) ? rorDisplay : names.get(0).getValue(); + } + + String value; + // set the value based on the configuration of the name selection type + switch (storedNameType) { + case ROR_DISPLAY : { + value = (rorDisplay != null) ? rorDisplay : label; + break; + } + case LOCALE_LABEL : { + value = label; + break; + } + default : { + value = enLabel != null ? enLabel : label; + } + } + + c.label = label; + c.value = value; + + c.extras.put("ror-id", authority); + + // set other-name, if exists, to show it in the UI as additional information about the institution + if (aliases.length() > 0) { + c.extras.put("other-names", aliases.toString()); + } + + if (!rorItem.getLocations().isEmpty()) { + Location location = rorItem.getLocations().get(0); + Location.GeonamesDetails geonamesDetails = location.getGeonamesDetails(); + if (geonamesDetails != null) { + c.extras.put("location", geonamesDetails.getName() + ", " + + geonamesDetails.getCountrySubdivisionName() + ", " + + geonamesDetails.getCountryName() + ", " + + geonamesDetails.getContinentName()); + } + } + + } + return c; + } + + private static StoredNameType resolveStoredNameType() { + ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); + return StoredNameType.fromString( + configurationService.getProperty("ror.authority.stored-name-type", "en_label")); + } + + private static String sanitizeQuery(String query) { + if (query.startsWith("\"") && query.endsWith("\"")) { + return query; + } else { + return "\"" + query + "\""; + } + } + + /** + * The type of the name that will be stored in the metadata, + * based on the configuration property "ror.authority.stored-name-type". + */ + private enum StoredNameType { + ROR_DISPLAY, + EN_LABEL, + LOCALE_LABEL; + + static StoredNameType fromString(String text) { + try { + return StoredNameType.valueOf(text.toUpperCase()); + } catch (IllegalArgumentException e) { + return EN_LABEL; + } + } + } + +} diff --git a/dspace-api/src/main/java/org/dspace/external/model/ror/Location.java b/dspace-api/src/main/java/org/dspace/external/model/ror/Location.java new file mode 100644 index 000000000000..3002a053aac9 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/external/model/ror/Location.java @@ -0,0 +1,79 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.external.model.ror; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Location model representing the single location element from ROR API response. + * + * @author Milan Kuchtiak + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class Location { + private final int geonamesId; + private final GeonamesDetails geonamesDetails; + + public Location(@JsonProperty("geonames_id") int id, + @JsonProperty("geonames_details") GeonamesDetails geonamesDetails) { + this.geonamesId = id; + this.geonamesDetails = geonamesDetails; + } + + public int getGeonamesId() { + return geonamesId; + } + + public GeonamesDetails getGeonamesDetails() { + return geonamesDetails; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class GeonamesDetails { + + private final String name; + private final String countrySubdivisionName; + private final String countryCode; + private final String countryName; + private final String continentName; + + public GeonamesDetails(@JsonProperty("name") String name, + @JsonProperty("country_subdivision_name") String countrySubdivisionName, + @JsonProperty("country_code") String countryCode, + @JsonProperty("country_name") String countryName, + @JsonProperty("continent_name") String continentName) { + this.name = name; + this.countrySubdivisionName = countrySubdivisionName; + this.countryCode = countryCode; + this.countryName = countryName; + this.continentName = continentName; + } + + public String getName() { + return name; + } + + public String getCountrySubdivisionName() { + return countrySubdivisionName; + } + + public String getCountryCode() { + return countryCode; + } + + public String getCountryName() { + return countryName; + } + + public String getContinentName() { + return continentName; + } + } + +} diff --git a/dspace-api/src/main/java/org/dspace/external/model/ror/RorItem.java b/dspace-api/src/main/java/org/dspace/external/model/ror/RorItem.java new file mode 100644 index 000000000000..fb2921343fe6 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/external/model/ror/RorItem.java @@ -0,0 +1,92 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.external.model.ror; + +import java.util.List; +import java.util.Optional; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * ROR item model representing the single item from ROR API response. + * + * @author Milan Kuchtiak + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class RorItem { + + private final String id; + private final List names; + private final String status; + private final String[] types; + private final List locations; + + @JsonCreator() + public RorItem(@JsonProperty("id") String id, + @JsonProperty("names") List names, + @JsonProperty("locations") List locations, + @JsonProperty("status") String status, + @JsonProperty("types") String[] types) { + this.id = id; + this.names = Optional.ofNullable(names).orElse(List.of()); + this.locations = Optional.ofNullable(locations).orElse(List.of()); + this.status = status; + this.types = Optional.ofNullable(types).orElse(new String[0]); + } + + public String getId() { + return id; + } + + public List getNames() { + return names; + } + + public List getLocations() { + return locations; + } + + public String getStatus() { + return status; + } + + public String[] getTypes() { + return types; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Name { + private String lang; + private List types; + private String value; + + @JsonCreator() + public Name(@JsonProperty("lang") String lang, + @JsonProperty("types") List types, + @JsonProperty("value") String value) { + this.lang = lang; + this.types = Optional.ofNullable(types).orElse(List.of()); + this.value = value; + } + + public String getLang() { + return lang; + } + + public List getTypes() { + return types; + } + + public String getValue() { + return value; + } + } + +} diff --git a/dspace-api/src/main/java/org/dspace/external/model/ror/RorItems.java b/dspace-api/src/main/java/org/dspace/external/model/ror/RorItems.java new file mode 100644 index 000000000000..7fdd38489faa --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/external/model/ror/RorItems.java @@ -0,0 +1,50 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.external.model.ror; + +import java.util.List; +import java.util.Optional; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * ROR items model representing the ROR API response. + * + * @author Milan Kuchtiak + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class RorItems { + + private final int noOfResults; + private final int timeTaken; + private final List items; + + @JsonCreator() + public RorItems(@JsonProperty("number_of_results") int noOfResults, + @JsonProperty("time_taken") int timeTaken, + @JsonProperty("items") List items + ) { + this.noOfResults = noOfResults; + this.timeTaken = timeTaken; + this.items = Optional.ofNullable(items).orElse(List.of()); + } + + public int getNoOfResults() { + return noOfResults; + } + + public int getTimeTaken() { + return timeTaken; + } + + public List getItems() { + return items; + } +} diff --git a/dspace-api/src/main/java/org/dspace/external/ror/CacheLogger.java b/dspace-api/src/main/java/org/dspace/external/ror/CacheLogger.java new file mode 100644 index 000000000000..2e47f1d6e7f7 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/external/ror/CacheLogger.java @@ -0,0 +1,27 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.external.ror; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ehcache.event.CacheEvent; +import org.ehcache.event.CacheEventListener; + +/** + * A simple logger for ROR label cache events + * + * @author Milan Kuchtiak + */ +public class CacheLogger implements CacheEventListener { + private static final Logger log = LogManager.getLogger(CacheLogger.class); + @Override + public void onEvent(CacheEvent event) { + log.debug("ROR Cache Event Type: {} | Key: {} | Old Value: {} | New Value: {}", + event.getType(), event.getKey(), event.getOldValue(), event.getNewValue()); + } +} diff --git a/dspace-api/src/test/data/dspaceFolder/config/spring/api/ror-authority-services.xml b/dspace-api/src/test/data/dspaceFolder/config/spring/api/ror-authority-services.xml new file mode 100644 index 000000000000..711c8d6ce645 --- /dev/null +++ b/dspace-api/src/test/data/dspaceFolder/config/spring/api/ror-authority-services.xml @@ -0,0 +1,15 @@ + + + + + + + + + diff --git a/dspace-api/src/test/java/org/dspace/external/MockRorRestConnector.java b/dspace-api/src/test/java/org/dspace/external/MockRorRestConnector.java new file mode 100644 index 000000000000..bc30f52110f4 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/external/MockRorRestConnector.java @@ -0,0 +1,69 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.external; + +import java.util.Optional; +import javax.ws.rs.ProcessingException; +import javax.ws.rs.core.Configuration; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.glassfish.jersey.message.internal.OutboundJaxrsResponse; +import org.glassfish.jersey.message.internal.OutboundMessageContext; + +/** + * Mock implementation of RorRestConnector for testing purposes. + * It returns predefined responses based on the input query or ID. + * + * @author Milan Kuchtiak + */ +public class MockRorRestConnector extends RorRestConnector { + + @Override + public Response getByQuery(String query, int page) { + if (query != null && query.startsWith("\"")) { + return getMockResponse("/org/dspace/external/ror/UniversityOfPisaByQueryExact.json"); + } else { + return getMockResponse("/org/dspace/external/ror/UniversityOfPisa.json"); + } + } + + @Override + public Response getByID(String id) { + if (id.matches(ROR_ID_PATTERN)) { + return getMockResponse("/org/dspace/external/ror/UniversityOfPisaByID.json"); + } else { + return Response.status(Response.Status.NOT_FOUND).build(); + } + } + + private static Response getMockResponse(String filePath) { + return new MockResponse<>(Response.Status.OK, + Optional.ofNullable(MockRorRestConnector.class.getResourceAsStream(filePath)) + .orElseThrow(() -> new IllegalStateException("Resource " + filePath + " not found."))); + } + + public static class MockResponse extends OutboundJaxrsResponse { + T responseBody; + + public MockResponse(Status status, T responseBody) { + super(status, new OutboundMessageContext((Configuration) null)); + this.responseBody = responseBody; + } + + @Override + public E readEntity(Class cls) throws ProcessingException { + return (E) responseBody; + } + + @Override + public MediaType getMediaType() { + return MediaType.APPLICATION_JSON_TYPE; + } + } +} diff --git a/dspace-api/src/test/resources/org/dspace/external/ror/UniversityOfPisa.json b/dspace-api/src/test/resources/org/dspace/external/ror/UniversityOfPisa.json new file mode 100644 index 000000000000..54040478d5e1 --- /dev/null +++ b/dspace-api/src/test/resources/org/dspace/external/ror/UniversityOfPisa.json @@ -0,0 +1,2172 @@ +{ + "number_of_results": 30133, + "time_taken": 59, + "items": [ + { + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2025-01-22", + "schema_version": "2.1" + } + }, + "domains": [ + "unipi.it" + ], + "established": 1343, + "external_ids": [ + { + "all": [ + "501100007514" + ], + "preferred": null, + "type": "fundref" + }, + { + "all": [ + "grid.5395.a" + ], + "preferred": "grid.5395.a", + "type": "grid" + }, + { + "all": [ + "0000 0004 1757 3729" + ], + "preferred": null, + "type": "isni" + }, + { + "all": [ + "Q645663" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/03ad39j10", + "links": [ + { + "type": "website", + "value": "https://www.unipi.it" + }, + { + "type": "wikipedia", + "value": "http://en.wikipedia.org/wiki/University_of_Pisa" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "IT", + "country_name": "Italy", + "country_subdivision_code": "52", + "country_subdivision_name": "Tuscany", + "lat": 43.70853, + "lng": 10.4036, + "name": "Pisa" + }, + "geonames_id": 3170647 + } + ], + "names": [ + { + "lang": null, + "types": [ + "acronym" + ], + "value": "UniPi" + }, + { + "lang": "en", + "types": [ + "ror_display", + "label" + ], + "value": "University of Pisa" + }, + { + "lang": "it", + "types": [ + "label" + ], + "value": "Università di Pisa" + }, + { + "lang": "de", + "types": [ + "label" + ], + "value": "Universität Pisa" + }, + { + "lang": "fr", + "types": [ + "label" + ], + "value": "Université de Pise" + } + ], + "relationships": [ + { + "label": "Ospedale Cisanello", + "type": "related", + "id": "https://ror.org/00mc91w09" + }, + { + "label": "Istituto Nazionale di Fisica Nucleare, Sezione di Pisa", + "type": "related", + "id": "https://ror.org/05symbg58" + } + ], + "status": "active", + "types": [ + "education", + "funder" + ] + }, + { + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2024-12-11", + "schema_version": "2.1" + } + }, + "domains": [], + "established": null, + "external_ids": [ + { + "all": [ + "grid.144189.1" + ], + "preferred": "grid.144189.1", + "type": "grid" + }, + { + "all": [ + "0000 0004 1756 8209" + ], + "preferred": null, + "type": "isni" + } + ], + "id": "https://ror.org/05xrcj819", + "links": [ + { + "type": "website", + "value": "http://www.ao-pisa.toscana.it/" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "IT", + "country_name": "Italy", + "country_subdivision_code": "52", + "country_subdivision_name": "Tuscany", + "lat": 43.70853, + "lng": 10.4036, + "name": "Pisa" + }, + "geonames_id": 3170647 + } + ], + "names": [ + { + "lang": "it", + "types": [ + "ror_display", + "label" + ], + "value": "Azienda Ospedaliera Universitaria Pisana" + }, + { + "lang": "en", + "types": [ + "label" + ], + "value": "University Hospital of Pisa" + } + ], + "relationships": [ + { + "label": "Ospedale Cisanello", + "type": "child", + "id": "https://ror.org/00mc91w09" + }, + { + "label": "ERN ReCONNET", + "type": "related", + "id": "https://ror.org/04069k268" + } + ], + "status": "active", + "types": [ + "healthcare" + ] + }, + { + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2024-12-11", + "schema_version": "2.1" + } + }, + "domains": [], + "established": 1992, + "external_ids": [ + { + "all": [ + "100007362", + "100007368" + ], + "preferred": null, + "type": "fundref" + }, + { + "all": [ + "grid.479041.f" + ], + "preferred": "grid.479041.f", + "type": "grid" + }, + { + "all": [ + "0000 0000 9587 6793" + ], + "preferred": null, + "type": "isni" + } + ], + "id": "https://ror.org/05jhnab13", + "links": [ + { + "type": "website", + "value": "http://www.fondazionepisa.it/" + }, + { + "type": "wikipedia", + "value": "https://en.wikipedia.org/wiki/Fondazione_Pisa" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "IT", + "country_name": "Italy", + "country_subdivision_code": "52", + "country_subdivision_name": "Tuscany", + "lat": 43.70853, + "lng": 10.4036, + "name": "Pisa" + }, + "geonames_id": 3170647 + } + ], + "names": [ + { + "lang": "it", + "types": [ + "alias" + ], + "value": "Fondazione Cassa di Risparmio di Pisa" + }, + { + "lang": "it", + "types": [ + "ror_display", + "label" + ], + "value": "Fondazione Pisa" + } + ], + "relationships": [], + "status": "active", + "types": [ + "funder", + "nonprofit" + ] + }, + { + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2025-01-22", + "schema_version": "2.1" + } + }, + "domains": [ + "pi.infn.it" + ], + "established": null, + "external_ids": [ + { + "all": [ + "grid.470216.6" + ], + "preferred": "grid.470216.6", + "type": "grid" + }, + { + "all": [ + "Q30265297" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/05symbg58", + "links": [ + { + "type": "website", + "value": "https://www.pi.infn.it" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "IT", + "country_name": "Italy", + "country_subdivision_code": "52", + "country_subdivision_name": "Tuscany", + "lat": 43.70853, + "lng": 10.4036, + "name": "Pisa" + }, + "geonames_id": 3170647 + } + ], + "names": [ + { + "lang": "it", + "types": [ + "alias" + ], + "value": "INFN Pisa" + }, + { + "lang": "en", + "types": [ + "alias" + ], + "value": "INFN Pisa Division" + }, + { + "lang": "en", + "types": [ + "alias" + ], + "value": "INFN Pisa Unit" + }, + { + "lang": "it", + "types": [ + "alias" + ], + "value": "INFN Sezione di Pisa" + }, + { + "lang": "it", + "types": [ + "acronym" + ], + "value": "INFN-PI" + }, + { + "lang": "it", + "types": [ + "label", + "ror_display" + ], + "value": "Istituto Nazionale di Fisica Nucleare, Sezione di Pisa" + }, + { + "lang": "en", + "types": [ + "label" + ], + "value": "National Institute for Nuclear Physics, Pisa Division" + } + ], + "relationships": [ + { + "label": "Istituto Nazionale di Fisica Nucleare", + "type": "parent", + "id": "https://ror.org/005ta0471" + }, + { + "label": "MAGIC Telescopes", + "type": "related", + "id": "https://ror.org/02w0r2764" + }, + { + "label": "University of Pisa", + "type": "related", + "id": "https://ror.org/03ad39j10" + } + ], + "status": "active", + "types": [ + "facility" + ] + }, + { + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2024-12-11", + "schema_version": "2.1" + } + }, + "domains": [], + "established": 1987, + "external_ids": [ + { + "all": [ + "grid.5740.6" + ], + "preferred": "grid.5740.6", + "type": "grid" + }, + { + "all": [ + "0000 0000 9120 5458" + ], + "preferred": null, + "type": "isni" + }, + { + "all": [ + "Q30252673" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/01t0n3b84", + "links": [ + { + "type": "website", + "value": "http://www.cpr.it/" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "IT", + "country_name": "Italy", + "country_subdivision_code": "52", + "country_subdivision_name": "Tuscany", + "lat": 43.70853, + "lng": 10.4036, + "name": "Pisa" + }, + "geonames_id": 3170647 + } + ], + "names": [ + { + "lang": "it", + "types": [ + "ror_display", + "label" + ], + "value": "Consorzio Pisa Ricerche" + } + ], + "relationships": [], + "status": "active", + "types": [ + "facility" + ] + }, + { + "admin": { + "created": { + "date": "2023-09-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2024-12-11", + "schema_version": "2.1" + } + }, + "domains": [], + "established": null, + "external_ids": [ + { + "all": [ + "0000 0004 1758 7813" + ], + "preferred": "0000 0004 1758 7813", + "type": "isni" + } + ], + "id": "https://ror.org/00vfm5970", + "links": [ + { + "type": "website", + "value": "https://www.pi.ingv.it" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "IT", + "country_name": "Italy", + "country_subdivision_code": "52", + "country_subdivision_name": "Tuscany", + "lat": 43.70853, + "lng": 10.4036, + "name": "Pisa" + }, + "geonames_id": 3170647 + } + ], + "names": [ + { + "lang": "it", + "types": [ + "ror_display", + "label" + ], + "value": "INGV Sezione di Pisa" + }, + { + "lang": null, + "types": [ + "acronym" + ], + "value": "INGV-PI" + }, + { + "lang": "it", + "types": [ + "alias" + ], + "value": "Istituto Nazionale di Geofisica e Vulcanologia Sezione di Pisa" + }, + { + "lang": "en", + "types": [ + "alias" + ], + "value": "National Institute of Geophysics and Volcanology, Pisa Section" + } + ], + "relationships": [ + { + "label": "Istituto Nazionale di Geofisica e Vulcanologia", + "type": "parent", + "id": "https://ror.org/00qps9a02" + } + ], + "status": "active", + "types": [ + "facility" + ] + }, + { + "admin": { + "created": { + "date": "2024-09-14", + "schema_version": "2.0" + }, + "last_modified": { + "date": "2024-12-11", + "schema_version": "2.1" + } + }, + "domains": [ + "liceodini.it" + ], + "established": 1924, + "external_ids": [ + { + "all": [ + "Q30889474" + ], + "preferred": "Q30889474", + "type": "wikidata" + } + ], + "id": "https://ror.org/006xg2x43", + "links": [ + { + "type": "website", + "value": "https://www.liceodini.it" + }, + { + "type": "wikipedia", + "value": "https://it.wikipedia.org/wiki/Liceo_scientifico_statale_Ulisse_Dini" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "IT", + "country_name": "Italy", + "country_subdivision_code": "52", + "country_subdivision_name": "Tuscany", + "lat": 43.70853, + "lng": 10.4036, + "name": "Pisa" + }, + "geonames_id": 3170647 + } + ], + "names": [ + { + "lang": "it", + "types": [ + "alias" + ], + "value": "Liceo Dini" + }, + { + "lang": "it", + "types": [ + "alias" + ], + "value": "Liceo Scientifico \"Ulisse Dini\"" + }, + { + "lang": "it", + "types": [ + "alias" + ], + "value": "Liceo Scientifico 'Ulisse Dini' - Pisa" + }, + { + "lang": "it", + "types": [ + "alias" + ], + "value": "Liceo Scientifico Ulisse Dini" + }, + { + "lang": "it", + "types": [ + "label", + "ror_display" + ], + "value": "Liceo scientifico statale Ulisse Dini" + }, + { + "lang": "it", + "types": [ + "alias" + ], + "value": "U. Dini" + }, + { + "lang": "en", + "types": [ + "label" + ], + "value": "Ulisse Dini Scientific High School" + } + ], + "relationships": [], + "status": "active", + "types": [ + "education" + ] + }, + { + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2024-12-11", + "schema_version": "2.1" + } + }, + "domains": [ + "sns.it" + ], + "established": 1810, + "external_ids": [ + { + "all": [ + "100009093" + ], + "preferred": null, + "type": "fundref" + }, + { + "all": [ + "grid.6093.c" + ], + "preferred": "grid.6093.c", + "type": "grid" + }, + { + "all": [ + "Q672416" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/03aydme10", + "links": [ + { + "type": "website", + "value": "https://www.sns.it" + }, + { + "type": "wikipedia", + "value": "http://en.wikipedia.org/wiki/Scuola_Normale_Superiore_di_Pisa" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "IT", + "country_name": "Italy", + "country_subdivision_code": "52", + "country_subdivision_name": "Tuscany", + "lat": 43.70853, + "lng": 10.4036, + "name": "Pisa" + }, + "geonames_id": 3170647 + } + ], + "names": [ + { + "lang": null, + "types": [ + "acronym" + ], + "value": "SNS" + }, + { + "lang": "it", + "types": [ + "ror_display", + "label" + ], + "value": "Scuola Normale Superiore" + }, + { + "lang": "it", + "types": [ + "alias" + ], + "value": "Scuola Normale Superiore di Pisa" + }, + { + "lang": "fr", + "types": [ + "alias" + ], + "value": "École Normale Supérieure de Pise" + } + ], + "relationships": [ + { + "label": "National Enterprise for NanoScience and NanoTechnology", + "type": "child", + "id": "https://ror.org/01sgfhb12" + } + ], + "status": "active", + "types": [ + "education", + "funder" + ] + }, + { + "admin": { + "created": { + "date": "2024-10-29", + "schema_version": "2.0" + }, + "last_modified": { + "date": "2024-12-11", + "schema_version": "2.1" + } + }, + "domains": [], + "established": null, + "external_ids": [], + "id": "https://ror.org/05etrbr47", + "links": [ + { + "type": "website", + "value": "https://web.infn.it/GC-Siena" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "IT", + "country_name": "Italy", + "country_subdivision_code": "52", + "country_subdivision_name": "Tuscany", + "lat": 43.31822, + "lng": 11.33064, + "name": "Siena" + }, + "geonames_id": 3166548 + } + ], + "names": [ + { + "lang": "it", + "types": [ + "alias" + ], + "value": "INFN Gruppo Collegato di Siena" + }, + { + "lang": "it", + "types": [ + "alias" + ], + "value": "INFN Gruppo Collegato di Siena a INFN-Pisa" + }, + { + "lang": "it", + "types": [ + "acronym" + ], + "value": "INFN-GCSI" + }, + { + "lang": "it", + "types": [ + "label", + "ror_display" + ], + "value": "Istituto Nazionale di Fisica Nucleare, Gruppo Collegato di Siena" + } + ], + "relationships": [ + { + "label": "Istituto Nazionale di Fisica Nucleare, Sezione di Firenze", + "type": "parent", + "id": "https://ror.org/02vv5y108" + }, + { + "label": "University of Siena", + "type": "related", + "id": "https://ror.org/01tevnk56" + } + ], + "status": "active", + "types": [ + "education", + "facility" + ] + }, + { + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2024-12-11", + "schema_version": "2.1" + } + }, + "domains": [], + "established": 1599, + "external_ids": [ + { + "all": [ + "grid.440820.a" + ], + "preferred": "grid.440820.a", + "type": "grid" + }, + { + "all": [ + "Q935460" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/006zjws59", + "links": [ + { + "type": "website", + "value": "http://www.uvic-ucc.cat/en" + }, + { + "type": "wikipedia", + "value": "https://en.wikipedia.org/wiki/University_of_Vic_-_Central_University_of_Catalonia" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "ES", + "country_name": "Spain", + "country_subdivision_code": "CT", + "country_subdivision_name": "Catalonia", + "lat": 41.93012, + "lng": 2.25486, + "name": "Vic" + }, + "geonames_id": 3106050 + } + ], + "names": [ + { + "lang": null, + "types": [ + "acronym" + ], + "value": "UVic-UCC" + }, + { + "lang": "es", + "types": [ + "alias" + ], + "value": "Universidad de Vic" + }, + { + "lang": "es", + "types": [ + "label" + ], + "value": "Universidad de Vic - Universidad Central de Catalunya" + }, + { + "lang": "es", + "types": [ + "alias" + ], + "value": "Universitat de Vic" + }, + { + "lang": "ca", + "types": [ + "ror_display", + "label" + ], + "value": "Universitat de Vic - Universitat Central de Catalunya" + }, + { + "lang": "en", + "types": [ + "alias" + ], + "value": "University of Vic" + }, + { + "lang": "en", + "types": [ + "label" + ], + "value": "University of Vic - Central University of Catalonia" + } + ], + "relationships": [], + "status": "active", + "types": [ + "education" + ] + }, + { + "admin": { + "created": { + "date": "2019-02-17", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2024-12-11", + "schema_version": "2.1" + } + }, + "domains": [], + "established": 1957, + "external_ids": [ + { + "all": [ + "grid.501720.1" + ], + "preferred": "grid.501720.1", + "type": "grid" + }, + { + "all": [ + "Q10829127" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/04bm3wy68", + "links": [ + { + "type": "website", + "value": "http://www.dhsphue.edu.vn" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "AS", + "continent_name": "Asia", + "country_code": "VN", + "country_name": "Vietnam", + "country_subdivision_code": "26", + "country_subdivision_name": "Thừa Thiên Huế Province", + "lat": 16.4619, + "lng": 107.59546, + "name": "Huế" + }, + "geonames_id": 1580240 + } + ], + "names": [ + { + "lang": "en", + "types": [ + "alias" + ], + "value": "Hue University" + }, + { + "lang": "en", + "types": [ + "label", + "ror_display" + ], + "value": "Hue University of Education" + }, + { + "lang": "en", + "types": [ + "alias" + ], + "value": "Hue University's College of Education" + }, + { + "lang": "vi", + "types": [ + "label" + ], + "value": "Trường Đại học Sư phạm Huế" + }, + { + "lang": "en", + "types": [ + "label" + ], + "value": "University of Education, Hue University" + } + ], + "relationships": [], + "status": "active", + "types": [ + "education" + ] + }, + { + "admin": { + "created": { + "date": "2025-10-27", + "schema_version": "2.1" + }, + "last_modified": { + "date": "2025-10-28", + "schema_version": "2.1" + } + }, + "domains": [ + "pnc.edu.ph" + ], + "established": 2003, + "external_ids": [ + { + "all": [ + "0000 0005 0599 0581" + ], + "preferred": "0000 0005 0599 0581", + "type": "isni" + }, + { + "all": [ + "Q7129041" + ], + "preferred": "Q7129041", + "type": "wikidata" + } + ], + "id": "https://ror.org/05h0cmr57", + "links": [ + { + "type": "wikipedia", + "value": "https://en.wikipedia.org/wiki/University_of_Cabuyao" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "AS", + "continent_name": "Asia", + "country_code": "PH", + "country_name": "Philippines", + "country_subdivision_code": "40", + "country_subdivision_name": "Calabarzon", + "lat": 14.2726, + "lng": 121.1262, + "name": "Cabuyao" + }, + "geonames_id": 1721281 + } + ], + "names": [ + { + "lang": "tl", + "types": [ + "label", + "ror_display" + ], + "value": "Pamantasan ng Cabuyao" + }, + { + "lang": "tl", + "types": [ + "acronym" + ], + "value": "PnC" + }, + { + "lang": "en", + "types": [ + "label" + ], + "value": "University of Cabuyao" + } + ], + "relationships": [], + "status": "active", + "types": [ + "education" + ] + }, + { + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2025-10-28", + "schema_version": "2.1" + } + }, + "domains": [ + "luguniv.edu.ua" + ], + "established": 1921, + "external_ids": [ + { + "all": [ + "grid.445812.e" + ], + "preferred": "grid.445812.e", + "type": "grid" + }, + { + "all": [ + "0000 0004 0489 542X" + ], + "preferred": null, + "type": "isni" + }, + { + "all": [ + "Q4267928" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/040wb2y55", + "links": [ + { + "type": "website", + "value": "https://luguniv.edu.ua" + }, + { + "type": "wikipedia", + "value": "https://en.wikipedia.org/wiki/University_of_Luhansk" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "UA", + "country_name": "Ukraine", + "country_subdivision_code": "09", + "country_subdivision_name": "Luhansk", + "lat": 48.56814, + "lng": 39.30553, + "name": "Luhansk" + }, + "geonames_id": 702658 + } + ], + "names": [ + { + "lang": null, + "types": [ + "acronym" + ], + "value": "LNU" + }, + { + "lang": "en", + "types": [ + "alias" + ], + "value": "Taras Shevchenko National University of Luhansk" + }, + { + "lang": null, + "types": [ + "ror_display", + "label" + ], + "value": "University of Luhansk" + }, + { + "lang": "pl", + "types": [ + "label" + ], + "value": "Ługański Uniwersytet Narodowy im. Tarasa Szewczenki" + }, + { + "lang": "ru", + "types": [ + "label" + ], + "value": "Луганский национальный университет имени Тараса Шевченко" + }, + { + "lang": "uk", + "types": [ + "label" + ], + "value": "Луганський національний університет імені Тараса Шевченка" + } + ], + "relationships": [], + "status": "active", + "types": [ + "education" + ] + }, + { + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2025-10-28", + "schema_version": "2.1" + } + }, + "domains": [ + "univ-parakou.bj" + ], + "established": 2001, + "external_ids": [ + { + "all": [ + "grid.440525.2" + ], + "preferred": "grid.440525.2", + "type": "grid" + }, + { + "all": [ + "0000 0004 0457 5047" + ], + "preferred": null, + "type": "isni" + }, + { + "all": [ + "Q3551659" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/025wndx93", + "links": [ + { + "type": "website", + "value": "https://www.univ-parakou.bj" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "AF", + "continent_name": "Africa", + "country_code": "BJ", + "country_name": "Benin", + "country_subdivision_code": "BO", + "country_subdivision_name": "Borgou", + "lat": 9.33716, + "lng": 2.63031, + "name": "Parakou" + }, + "geonames_id": 2392204 + } + ], + "names": [ + { + "lang": null, + "types": [ + "acronym" + ], + "value": "UP" + }, + { + "lang": "en", + "types": [ + "label" + ], + "value": "University of Parakou" + }, + { + "lang": "fr", + "types": [ + "ror_display", + "label" + ], + "value": "Université de Parakou" + } + ], + "relationships": [], + "status": "active", + "types": [ + "education" + ] + }, + { + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2025-10-28", + "schema_version": "2.1" + } + }, + "domains": [ + "univ-bangui.org" + ], + "established": 1969, + "external_ids": [ + { + "all": [ + "grid.25077.37" + ], + "preferred": "grid.25077.37", + "type": "grid" + }, + { + "all": [ + "0000 0000 9737 7808" + ], + "preferred": null, + "type": "isni" + }, + { + "all": [ + "Q1638914" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/020q46z35", + "links": [ + { + "type": "website", + "value": "https://www.univ-bangui.org/" + }, + { + "type": "wikipedia", + "value": "https://en.wikipedia.org/wiki/University_of_Bangui" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "AF", + "continent_name": "Africa", + "country_code": "CF", + "country_name": "Central African Republic", + "country_subdivision_code": "BGF", + "country_subdivision_name": "Bangui", + "lat": 4.36122, + "lng": 18.55496, + "name": "Bangui" + }, + "geonames_id": 2389853 + } + ], + "names": [ + { + "lang": "en", + "types": [ + "ror_display", + "label" + ], + "value": "University of Bangui" + }, + { + "lang": "fr", + "types": [ + "label" + ], + "value": "Université de Bangui" + } + ], + "relationships": [], + "status": "active", + "types": [ + "education" + ] + }, + { + "admin": { + "created": { + "date": "2025-06-24", + "schema_version": "2.1" + }, + "last_modified": { + "date": "2025-10-28", + "schema_version": "2.1" + } + }, + "domains": [ + "uomanara.edu.iq" + ], + "established": 2017, + "external_ids": [], + "id": "https://ror.org/04k20kq32", + "links": [ + { + "type": "website", + "value": "https://uomanara.edu.iq" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "AS", + "continent_name": "Asia", + "country_code": "IQ", + "country_name": "Iraq", + "country_subdivision_code": "MA", + "country_subdivision_name": "Maysan", + "lat": 31.9, + "lng": 47.06667, + "name": "Maysan" + }, + "geonames_id": 93540 + } + ], + "names": [ + { + "lang": "en", + "types": [ + "alias" + ], + "value": "Al-Manara College for Medical Sciences" + }, + { + "lang": "en", + "types": [ + "alias" + ], + "value": "Al-Manara University" + }, + { + "lang": "en", + "types": [ + "label", + "ror_display" + ], + "value": "University of Manara" + }, + { + "lang": "ar", + "types": [ + "label" + ], + "value": "جامعة المنارة" + }, + { + "lang": "ar", + "types": [ + "alias" + ], + "value": "لكلية المنارة للعلوم الطبية" + } + ], + "relationships": [], + "status": "active", + "types": [ + "education" + ] + }, + { + "admin": { + "created": { + "date": "2020-03-15", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2025-10-28", + "schema_version": "2.1" + } + }, + "domains": [ + "uok.ac.rw" + ], + "established": 2013, + "external_ids": [ + { + "all": [ + "grid.507637.0" + ], + "preferred": "grid.507637.0", + "type": "grid" + }, + { + "all": [ + "0000 0004 4676 8461" + ], + "preferred": null, + "type": "isni" + }, + { + "all": [ + "Q48773691" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/03v842g47", + "links": [ + { + "type": "website", + "value": "https://uok.ac.rw" + }, + { + "type": "wikipedia", + "value": "https://en.wikipedia.org/wiki/University_of_Kigali" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "AF", + "continent_name": "Africa", + "country_code": "RW", + "country_name": "Rwanda", + "country_subdivision_code": "01", + "country_subdivision_name": "Kigali", + "lat": -1.94995, + "lng": 30.05885, + "name": "Kigali" + }, + "geonames_id": 202061 + } + ], + "names": [ + { + "lang": "en", + "types": [ + "ror_display", + "label" + ], + "value": "University of Kigali" + }, + { + "lang": null, + "types": [ + "acronym" + ], + "value": "UoK" + } + ], + "relationships": [], + "status": "active", + "types": [ + "education" + ] + }, + { + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2025-10-28", + "schema_version": "2.1" + } + }, + "domains": [ + "unifa.ac.id" + ], + "established": 2008, + "external_ids": [ + { + "all": [ + "grid.443675.7" + ], + "preferred": "grid.443675.7", + "type": "grid" + }, + { + "all": [ + "0000 0004 0386 0305" + ], + "preferred": null, + "type": "isni" + }, + { + "all": [ + "Q23807190" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/05whqt140", + "links": [ + { + "type": "website", + "value": "https://unifa.ac.id" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "AS", + "continent_name": "Asia", + "country_code": "ID", + "country_name": "Indonesia", + "country_subdivision_code": "SN", + "country_subdivision_name": "South Sulawesi", + "lat": -5.14861, + "lng": 119.43194, + "name": "Makassar" + }, + "geonames_id": 1622786 + } + ], + "names": [ + { + "lang": null, + "types": [ + "acronym" + ], + "value": "UNIFA" + }, + { + "lang": "id", + "types": [ + "ror_display", + "label" + ], + "value": "Universitas Fajar" + }, + { + "lang": "en", + "types": [ + "alias" + ], + "value": "University of Dawn" + } + ], + "relationships": [], + "status": "active", + "types": [ + "education" + ] + }, + { + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2025-10-28", + "schema_version": "2.1" + } + }, + "domains": [ + "uni.lu" + ], + "established": 2003, + "external_ids": [ + { + "all": [ + "100008665" + ], + "preferred": null, + "type": "fundref" + }, + { + "all": [ + "grid.16008.3f" + ], + "preferred": "grid.16008.3f", + "type": "grid" + }, + { + "all": [ + "0000 0001 2295 9843" + ], + "preferred": null, + "type": "isni" + }, + { + "all": [ + "Q59668" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/036x5ad56", + "links": [ + { + "type": "website", + "value": "https://www.uni.lu" + }, + { + "type": "wikipedia", + "value": "http://en.wikipedia.org/wiki/University_of_Luxembourg" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "LU", + "country_name": "Luxembourg", + "country_subdivision_code": "LU", + "country_subdivision_name": "Luxembourg", + "lat": 49.60982, + "lng": 6.13268, + "name": "Luxembourg" + }, + "geonames_id": 2960316 + } + ], + "names": [ + { + "lang": "en", + "types": [ + "ror_display", + "label" + ], + "value": "University of Luxembourg" + }, + { + "lang": "de", + "types": [ + "label" + ], + "value": "Universität Luxemburg" + }, + { + "lang": "fr", + "types": [ + "label" + ], + "value": "Université du Luxembourg" + } + ], + "relationships": [ + { + "label": "Luxembourg Centre for Contemporary and Digital History", + "type": "child", + "id": "https://ror.org/054b6pr16" + }, + { + "label": "Luxembourg Centre for Systems Biomedicine", + "type": "child", + "id": "https://ror.org/051tr1y59" + }, + { + "label": "Interdisciplinary Centre for Security, Reliability and Trust", + "type": "child", + "id": "https://ror.org/02qav7c04" + }, + { + "label": "Luxembourg Centre for Socio-Environmental Systems", + "type": "child", + "id": "https://ror.org/04c6fdf96" + }, + { + "label": "Luxembourg Centre for European Law", + "type": "child", + "id": "https://ror.org/04cqhc152" + }, + { + "label": "Centre Hospitalier de Luxembourg", + "type": "related", + "id": "https://ror.org/03xq7w797" + } + ], + "status": "active", + "types": [ + "education", + "funder" + ] + }, + { + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2025-10-28", + "schema_version": "2.1" + } + }, + "domains": [ + "uma.pt" + ], + "established": 1988, + "external_ids": [ + { + "all": [ + "501100013990" + ], + "preferred": "501100013990", + "type": "fundref" + }, + { + "all": [ + "grid.26793.39" + ], + "preferred": "grid.26793.39", + "type": "grid" + }, + { + "all": [ + "0000 0001 2155 1272" + ], + "preferred": null, + "type": "isni" + }, + { + "all": [ + "Q1434847" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/0442zbe52", + "links": [ + { + "type": "website", + "value": "https://www.uma.pt/" + }, + { + "type": "wikipedia", + "value": "http://en.wikipedia.org/wiki/University_of_Madeira" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "PT", + "country_name": "Portugal", + "country_subdivision_code": "30", + "country_subdivision_name": "Madeira", + "lat": 32.66568, + "lng": -16.92547, + "name": "Funchal" + }, + "geonames_id": 2267827 + } + ], + "names": [ + { + "lang": null, + "types": [ + "acronym" + ], + "value": "UMa" + }, + { + "lang": "pt", + "types": [ + "ror_display", + "label" + ], + "value": "Universidade da Madeira" + }, + { + "lang": "en", + "types": [ + "label" + ], + "value": "University of Madeira" + } + ], + "relationships": [ + { + "label": "Centro de Investigação de Matemática e Aplicações", + "type": "child", + "id": "https://ror.org/058k6gb21" + }, + { + "label": "Centro de Investigação em Educação", + "type": "child", + "id": "https://ror.org/00wdyvz26" + }, + { + "label": "Madeira N-Lincs", + "type": "child", + "id": "https://ror.org/04kt8mw18" + }, + { + "label": "Centro de Ciências Matemáticas", + "type": "child", + "id": "https://ror.org/04ycf0k71" + }, + { + "label": "Centro de Investigação em Estudos Regionais e Locais", + "type": "child", + "id": "https://ror.org/01551d523" + }, + { + "label": "Centro de Química da Madeira", + "type": "child", + "id": "https://ror.org/01tgdcv71" + }, + { + "label": "Grupo de Astronomia", + "type": "child", + "id": "https://ror.org/01hbpef95" + }, + { + "label": "ISOPlexis Banco de Germoplasma", + "type": "child", + "id": "https://ror.org/02c4ps936" + } + ], + "status": "active", + "types": [ + "education", + "funder" + ] + } + ], + "meta": { + "types": [ + { + "id": "education", + "title": "education", + "count": 15003 + }, + { + "id": "funder", + "title": "funder", + "count": 7064 + }, + { + "id": "facility", + "title": "facility", + "count": 4233 + }, + { + "id": "government", + "title": "government", + "count": 3315 + }, + { + "id": "nonprofit", + "title": "nonprofit", + "count": 2599 + }, + { + "id": "healthcare", + "title": "healthcare", + "count": 2088 + }, + { + "id": "other", + "title": "other", + "count": 1847 + }, + { + "id": "archive", + "title": "archive", + "count": 733 + }, + { + "id": "company", + "title": "company", + "count": 362 + } + ], + "countries": [ + { + "id": "us", + "title": "United States", + "count": 5564 + }, + { + "id": "cn", + "title": "China", + "count": 2749 + }, + { + "id": "jp", + "title": "Japan", + "count": 1872 + }, + { + "id": "in", + "title": "India", + "count": 1763 + }, + { + "id": "ru", + "title": "Russia", + "count": 1559 + }, + { + "id": "gb", + "title": "United Kingdom", + "count": 871 + }, + { + "id": "kr", + "title": "South Korea", + "count": 829 + }, + { + "id": "de", + "title": "Germany", + "count": 702 + }, + { + "id": "fr", + "title": "France", + "count": 702 + }, + { + "id": "ca", + "title": "Canada", + "count": 608 + } + ], + "continents": [ + { + "id": "as", + "title": "Asia", + "count": 11352 + }, + { + "id": "eu", + "title": "Europe", + "count": 9012 + }, + { + "id": "na", + "title": "North America", + "count": 6649 + }, + { + "id": "af", + "title": "Africa", + "count": 1816 + }, + { + "id": "sa", + "title": "South America", + "count": 826 + }, + { + "id": "oc", + "title": "Oceania", + "count": 479 + } + ], + "statuses": [ + { + "id": "active", + "title": "active", + "count": 30133 + } + ] + } +} \ No newline at end of file diff --git a/dspace-api/src/test/resources/org/dspace/external/ror/UniversityOfPisaByID.json b/dspace-api/src/test/resources/org/dspace/external/ror/UniversityOfPisaByID.json new file mode 100644 index 000000000000..84f2c738492f --- /dev/null +++ b/dspace-api/src/test/resources/org/dspace/external/ror/UniversityOfPisaByID.json @@ -0,0 +1,128 @@ +{ + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2025-01-22", + "schema_version": "2.1" + } + }, + "domains": [ + "unipi.it" + ], + "established": 1343, + "external_ids": [ + { + "all": [ + "501100007514" + ], + "preferred": null, + "type": "fundref" + }, + { + "all": [ + "grid.5395.a" + ], + "preferred": "grid.5395.a", + "type": "grid" + }, + { + "all": [ + "0000 0004 1757 3729" + ], + "preferred": null, + "type": "isni" + }, + { + "all": [ + "Q645663" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/03ad39j10", + "links": [ + { + "type": "website", + "value": "https://www.unipi.it" + }, + { + "type": "wikipedia", + "value": "http://en.wikipedia.org/wiki/University_of_Pisa" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "IT", + "country_name": "Italy", + "country_subdivision_code": "52", + "country_subdivision_name": "Tuscany", + "lat": 43.70853, + "lng": 10.4036, + "name": "Pisa" + }, + "geonames_id": 3170647 + } + ], + "names": [ + { + "lang": null, + "types": [ + "acronym" + ], + "value": "UniPi" + }, + { + "lang": "en", + "types": [ + "ror_display", + "label" + ], + "value": "University of Pisa" + }, + { + "lang": "it", + "types": [ + "label" + ], + "value": "Università di Pisa" + }, + { + "lang": "de", + "types": [ + "label" + ], + "value": "Universität Pisa" + }, + { + "lang": "fr", + "types": [ + "label" + ], + "value": "Université de Pise" + } + ], + "relationships": [ + { + "label": "Ospedale Cisanello", + "type": "related", + "id": "https://ror.org/00mc91w09" + }, + { + "label": "Istituto Nazionale di Fisica Nucleare, Sezione di Pisa", + "type": "related", + "id": "https://ror.org/05symbg58" + } + ], + "status": "active", + "types": [ + "education", + "funder" + ] +} \ No newline at end of file diff --git a/dspace-api/src/test/resources/org/dspace/external/ror/UniversityOfPisaByQueryExact.json b/dspace-api/src/test/resources/org/dspace/external/ror/UniversityOfPisaByQueryExact.json new file mode 100644 index 000000000000..d4cbd31353b7 --- /dev/null +++ b/dspace-api/src/test/resources/org/dspace/external/ror/UniversityOfPisaByQueryExact.json @@ -0,0 +1,169 @@ +{ + "number_of_results": 1, + "time_taken": 7, + "items": [ + { + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2025-01-22", + "schema_version": "2.1" + } + }, + "domains": [ + "unipi.it" + ], + "established": 1343, + "external_ids": [ + { + "all": [ + "501100007514" + ], + "preferred": null, + "type": "fundref" + }, + { + "all": [ + "grid.5395.a" + ], + "preferred": "grid.5395.a", + "type": "grid" + }, + { + "all": [ + "0000 0004 1757 3729" + ], + "preferred": null, + "type": "isni" + }, + { + "all": [ + "Q645663" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/03ad39j10", + "links": [ + { + "type": "website", + "value": "https://www.unipi.it" + }, + { + "type": "wikipedia", + "value": "http://en.wikipedia.org/wiki/University_of_Pisa" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "IT", + "country_name": "Italy", + "country_subdivision_code": "52", + "country_subdivision_name": "Tuscany", + "lat": 43.70853, + "lng": 10.4036, + "name": "Pisa" + }, + "geonames_id": 3170647 + } + ], + "names": [ + { + "lang": null, + "types": [ + "acronym" + ], + "value": "UniPi" + }, + { + "lang": "en", + "types": [ + "ror_display", + "label" + ], + "value": "University of Pisa" + }, + { + "lang": "it", + "types": [ + "label" + ], + "value": "Università di Pisa" + }, + { + "lang": "de", + "types": [ + "label" + ], + "value": "Universität Pisa" + }, + { + "lang": "fr", + "types": [ + "label" + ], + "value": "Université de Pise" + } + ], + "relationships": [ + { + "label": "Ospedale Cisanello", + "type": "related", + "id": "https://ror.org/00mc91w09" + }, + { + "label": "Istituto Nazionale di Fisica Nucleare, Sezione di Pisa", + "type": "related", + "id": "https://ror.org/05symbg58" + } + ], + "status": "active", + "types": [ + "education", + "funder" + ] + } + ], + "meta": { + "types": [ + { + "id": "education", + "title": "education", + "count": 1 + }, + { + "id": "funder", + "title": "funder", + "count": 1 + } + ], + "countries": [ + { + "id": "it", + "title": "Italy", + "count": 1 + } + ], + "continents": [ + { + "id": "eu", + "title": "Europe", + "count": 1 + } + ], + "statuses": [ + { + "id": "active", + "title": "active", + "count": 1 + } + ] + } +} \ No newline at end of file diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/VocabularyEntryLinkRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/VocabularyEntryLinkRepositoryIT.java new file mode 100644 index 000000000000..644a3cd65202 --- /dev/null +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/VocabularyEntryLinkRepositoryIT.java @@ -0,0 +1,193 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.Objects; + +import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.core.factory.CoreServiceFactory; +import org.dspace.services.ConfigurationService; +import org.dspace.services.factory.DSpaceServicesFactory; +import org.hamcrest.Matchers; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.springframework.test.web.servlet.ResultActions; + +public class VocabularyEntryLinkRepositoryIT extends AbstractControllerIntegrationTest { + + private static final String BASE_VOCABULARY_URL = "/api/submission/vocabularies"; + private static final String ROR_AUTHORITY_ENTRIES_URL = BASE_VOCABULARY_URL + "/SimpleRORAuthority/entries"; + private static final int MOCK_TOTAL_ELEMENTS = 30133; + private static final String CHOICE_AUTHORITY_PLUGIN_KEY = + "plugin.named.org.dspace.content.authority.ChoiceAuthority"; + + private static String[] originalChoiceAuthorities; + + @BeforeClass + public static void beforeClass() { + ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); + originalChoiceAuthorities = configurationService.getArrayProperty(CHOICE_AUTHORITY_PLUGIN_KEY); + configurationService.setProperty(CHOICE_AUTHORITY_PLUGIN_KEY, + new String[] { + "org.dspace.content.authority.SimpleRORAuthority = SimpleRORAuthority" + }); + CoreServiceFactory.getInstance().getPluginService().clearNamedPluginClasses(); + } + + @AfterClass + public static void afterClass() { + // restore the original ChoiceAuthority plugin configuration so this class does not + // leak the SimpleRORAuthority registration into other integration tests + ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); + configurationService.setProperty(CHOICE_AUTHORITY_PLUGIN_KEY, originalChoiceAuthorities); + CoreServiceFactory.getInstance().getPluginService().clearNamedPluginClasses(); + } + + @Test + public void rorAuthoritySizeNotDivisorOf20() throws Exception { + getClient().perform(get(ROR_AUTHORITY_ENTRIES_URL) + .param("filter", "University") + .param("size", "3")) + .andExpect(status().isBadRequest()) + .andExpect(result -> Assert.assertEquals( + "The page size must be a divisor of 20.", + Objects.requireNonNull(result.getResolvedException()).getMessage())); + } + + @Test + public void rorAuthorityTooManyPages() throws Exception { + getClient().perform(get(ROR_AUTHORITY_ENTRIES_URL) + .param("filter", "University") + .param("page", "500")) + .andExpect(status().isBadRequest()) + .andExpect(result -> Assert.assertEquals( + "Exceeded maximal page number for the ROR API, which is 499, for page size 20.", + Objects.requireNonNull(result.getResolvedException()).getMessage())); + } + + @Test + public void rorAuthorityTooManyPagesForSize4() throws Exception { + getClient().perform(get(ROR_AUTHORITY_ENTRIES_URL) + .param("filter", "University") + .param("size", "4") + .param("page", "2500")) + .andExpect(status().isBadRequest()) + .andExpect(result -> Assert.assertEquals( + "Exceeded maximal page number for the ROR API, which is 2499, for page size 4.", + Objects.requireNonNull(result.getResolvedException()).getMessage())); + } + + @Test + public void rorAuthorityRequestWithEntryID() throws Exception { + checkSingleItemResponse(getClient().perform(get(ROR_AUTHORITY_ENTRIES_URL) + .param("entryID", "03ad39j10")), "University of Pisa", "University of Pisa"); + } + + @Test + public void rorAuthorityRequestWithBadEntryID() throws Exception { + getClient().perform(get(ROR_AUTHORITY_ENTRIES_URL) + .param("entryID", "wrong_entry_id")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.entries", Matchers.hasSize(0))) + .andExpect(jsonPath("$.page.size", Matchers.is(20))) + .andExpect(jsonPath("$.page.number", Matchers.is(0))) + .andExpect(jsonPath("$.page.totalElements", Matchers.is(0))) + .andExpect(jsonPath("$.page.totalPages", Matchers.is(0))); + } + + @Test + public void rorAuthorityRequestWithQueryExact() throws Exception { + checkSingleItemResponse(getClient().perform(get(ROR_AUTHORITY_ENTRIES_URL) + .param("filter", "University of Pisa") + .param("exact", "true")), "University of Pisa", "University of Pisa"); + } + + @Test + public void rorAuthorityRequestWithResponseInLocale() throws Exception { + ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); + String defaultLocale = configurationService.getProperty("default.locale"); + String originalStoredNameType = configurationService.getProperty("ror.authority.stored-name-type", "en_label"); + configurationService.setProperty("default.locale", "it"); + configurationService.setProperty("ror.authority.stored-name-type", "locale_label"); + + try { + checkSingleItemResponse(getClient().perform(get(ROR_AUTHORITY_ENTRIES_URL) + .param("filter", "University of Pisa") + .param("exact", "true")), "Università di Pisa", "Università di Pisa"); + } finally { + configurationService.setProperty("default.locale", defaultLocale); + configurationService.setProperty("ror.authority.stored-name-type", originalStoredNameType); + } + } + + @Test + public void rorAuthorityRequestWithRorDisplaySelectionType() throws Exception { + ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); + String defaultLocale = configurationService.getProperty("default.locale"); + String originalStoredNameType = configurationService.getProperty("ror.authority.stored-name-type", "en_label"); + configurationService.setProperty("default.locale", "it"); + configurationService.setProperty("ror.authority.stored-name-type", "ror_display"); + + try { + checkSingleItemResponse(getClient().perform(get(ROR_AUTHORITY_ENTRIES_URL) + .param("filter", "University of Pisa") + .param("exact", "true")), "University of Pisa", "Università di Pisa"); + } finally { + configurationService.setProperty("default.locale", defaultLocale); + configurationService.setProperty("ror.authority.stored-name-type", originalStoredNameType); + } + } + + @Test + public void rorAuthorityRequestWithQuery() throws Exception { + getClient().perform(get(ROR_AUTHORITY_ENTRIES_URL) + .param("filter", "University of Pisa")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.entries", Matchers.hasSize(20))) + .andExpect(jsonPath("$.page.size", Matchers.is(20))) + .andExpect(jsonPath("$.page.number", Matchers.is(0))) + .andExpect(jsonPath("$.page.totalElements", Matchers.is(MOCK_TOTAL_ELEMENTS))) + .andExpect(jsonPath("$.page.totalPages", Matchers.is(MOCK_TOTAL_ELEMENTS / 20 + 1))); + } + + @Test + public void rorAuthorityRequestWithQueryAndPagination() throws Exception { + getClient().perform(get(ROR_AUTHORITY_ENTRIES_URL) + .param("filter", "University of Pisa") + .param("size", "4") + .param("page", "2000")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.entries", Matchers.hasSize(4))) + .andExpect(jsonPath("$.page.size", Matchers.is(4))) + .andExpect(jsonPath("$.page.number", Matchers.is(2000))) + .andExpect(jsonPath("$.page.totalElements", Matchers.is(MOCK_TOTAL_ELEMENTS))) + .andExpect(jsonPath("$.page.totalPages", Matchers.is(MOCK_TOTAL_ELEMENTS / 4 + 1))); + } + + private void checkSingleItemResponse(ResultActions resultActions, String expectedValue, String expectedDisplay) + throws Exception { + resultActions.andExpect(status().isOk()) + .andExpect(jsonPath("$._embedded.entries", Matchers.hasSize(1))) + .andExpect(jsonPath("$.page.size", Matchers.is(20))) + .andExpect(jsonPath("$.page.number", Matchers.is(0))) + .andExpect(jsonPath("$.page.totalElements", Matchers.is(1))) + .andExpect(jsonPath("$.page.totalPages", Matchers.is(1))) + .andExpect(jsonPath("$._embedded.entries[0].authority", Matchers.is("03ad39j10"))) + .andExpect(jsonPath("$._embedded.entries[0].display", Matchers.is(expectedDisplay))) + .andExpect(jsonPath("$._embedded.entries[0].value", Matchers.is(expectedValue))) + .andExpect(jsonPath("$._embedded.entries[0].otherInformation.location", + Matchers.is("Pisa, Tuscany, Italy, Europe"))); + } + +} diff --git a/dspace/config/ehcache.xml b/dspace/config/ehcache.xml index 15e6f85ba912..82717c25aefe 100644 --- a/dspace/config/ehcache.xml +++ b/dspace/config/ehcache.xml @@ -82,9 +82,31 @@ + + + 1 + + + + org.dspace.external.ror.CacheLogger + ASYNCHRONOUS + UNORDERED + CREATED + EXPIRED + REMOVED + EVICTED + + + + 1000 + 10 + + + + \ No newline at end of file diff --git a/dspace/config/features/enable-ror.cfg b/dspace/config/features/enable-ror.cfg new file mode 100644 index 000000000000..9e2ca42fb497 --- /dev/null +++ b/dspace/config/features/enable-ror.cfg @@ -0,0 +1,31 @@ +## Register the ROR authority plugin +plugin.named.org.dspace.content.authority.ChoiceAuthority = \ + org.dspace.content.authority.SimpleRORAuthority = SimpleRORAuthority + +choices.plugin.dc.publisher = SimpleRORAuthority +choices.presentation.dc.publisher = lookup +authority.controlled.dc.publisher = true + +ror.api-url = https://api.ror.org/v2/organizations + +### Add the following lines to local.cfg: +#include = features/enable-ror.cfg +#ror.client-id = <> +#ror.authority.stored-name-type = ror_display | en_label | locale_label + +# Notes: +# To obtain a ROR API Client ID, you need to register at: https://ror.org/api-client-id. +# The "ror.authority.stored-name-type" property defines how the authority value is selected +# from the ROR API response. The authority value is then stored to metadata field, e.g. to "dc.publisher" field. +# The ROR API returns multiple names based on the locale and the name type +# the name types are "ror_display", "label", "alias" and "acronym". +# For more information on the name types, see: https://ror.readme.io/docs/ror-data-structure +# Allowed values for the "ror.authority.stored-name-type" property are: +# "ror_display" - authority value is selected from the name of type "ror_display" +# "en_label" - authority value is selected from the name of type "label" with locale "en" +# "locale_label" - authority value is selected from the name of type "label" with locale matching the DSpace locale +# The default "ror.authority.stored-name-type" is "en_label". + +# To enable the SimpleRORAuthority choice working in the Item Submission Page(UI), for the publisher field, +# the following input-type should be set for the "dc.publisher" field, in the submission-forms.xml configuration file: +#onebox diff --git a/dspace/config/spring/api/ror-authority-services.xml b/dspace/config/spring/api/ror-authority-services.xml new file mode 100644 index 000000000000..3c9945533a7b --- /dev/null +++ b/dspace/config/spring/api/ror-authority-services.xml @@ -0,0 +1,23 @@ + + + + + + + + + + +