diff --git a/src/app/core/shared/bitstream.model.ts b/src/app/core/shared/bitstream.model.ts index 29a80069db9..c5d200483fa 100644 --- a/src/app/core/shared/bitstream.model.ts +++ b/src/app/core/shared/bitstream.model.ts @@ -13,6 +13,8 @@ import { } from '../cache/builders/build-decorators'; import { RemoteData } from '../data/remote-data'; import { BITSTREAM } from './bitstream.resource-type'; +import { BitstreamChecksum } from './bitstream-checksum.model'; +import { BITSTREAM_CHECKSUM } from './bitstream-checksum.resource'; import { BitstreamFormat } from './bitstream-format.model'; import { BITSTREAM_FORMAT } from './bitstream-format.resource-type'; import { Bundle } from './bundle.model'; @@ -21,6 +23,9 @@ import { ChildHALResource } from './child-hal-resource.model'; import { DSpaceObject } from './dspace-object.model'; import { HALLink } from './hal-link.model'; +// Store number if the bitstream is stored in the both stores (S3 and local) +export const SYNCHRONIZED_STORES_NUMBER = 77; + @typedObject @inheritSerialization(DSpaceObject) export class Bitstream extends DSpaceObject implements ChildHALResource { @@ -44,6 +49,12 @@ export class Bitstream extends DSpaceObject implements ChildHALResource { @autoserialize bundleName: string; + /** + * The number of the store where the bitstream is store, it could be S3, local or both. + */ + @autoserialize + storeNumber: number; + /** * The {@link HALLink}s for this Bitstream */ @@ -54,6 +65,10 @@ export class Bitstream extends DSpaceObject implements ChildHALResource { format: HALLink; content: HALLink; thumbnail: HALLink; + /** + * Optional: only the CLARIN backend exposes this link. + */ + checksum?: HALLink; accessStatus: HALLink; }; @@ -78,6 +93,12 @@ export class Bitstream extends DSpaceObject implements ChildHALResource { @link(BUNDLE) bundle?: Observable>; + /** + * The checksum values fetched from the DB, local and S3 store. + */ + @link(BITSTREAM_CHECKSUM) + checksum?: Observable>; + /** * The access status for this Bitstream * Will be undefined unless the access status {@link HALLink} has been resolved. diff --git a/src/app/item-page/edit-item-page/item-bitstreams/item-bitstreams.service.stub.ts b/src/app/item-page/edit-item-page/item-bitstreams/item-bitstreams.service.stub.ts index eb99c214ca4..cba98e497e1 100644 --- a/src/app/item-page/edit-item-page/item-bitstreams/item-bitstreams.service.stub.ts +++ b/src/app/item-page/edit-item-page/item-bitstreams/item-bitstreams.service.stub.ts @@ -44,10 +44,11 @@ export class ItemBitstreamsServiceStub { getColumnSizes = jasmine.createSpy('getColumnSizes').and .returnValue( new ResponsiveTableSizes([ - new ResponsiveColumnSizes(2, 2, 3, 4, 4), - new ResponsiveColumnSizes(2, 3, 3, 3, 3), + new ResponsiveColumnSizes(3, 3, 3, 3, 3), + new ResponsiveColumnSizes(3, 3, 3, 3, 3), + new ResponsiveColumnSizes(2, 2, 2, 2, 2), + new ResponsiveColumnSizes(2, 2, 2, 2, 2), new ResponsiveColumnSizes(2, 2, 2, 2, 2), - new ResponsiveColumnSizes(6, 5, 4, 3, 3), ]), ); diff --git a/src/app/item-page/edit-item-page/item-bitstreams/item-bitstreams.service.ts b/src/app/item-page/edit-item-page/item-bitstreams/item-bitstreams.service.ts index 5cb25c6537f..2f7d84f5ca5 100644 --- a/src/app/item-page/edit-item-page/item-bitstreams/item-bitstreams.service.ts +++ b/src/app/item-page/edit-item-page/item-bitstreams/item-bitstreams.service.ts @@ -383,13 +383,15 @@ export class ItemBitstreamsService { getColumnSizes(): ResponsiveTableSizes { return new ResponsiveTableSizes([ // Name column - new ResponsiveColumnSizes(2, 2, 3, 4, 4), + new ResponsiveColumnSizes(3, 3, 3, 3, 3), // Description column - new ResponsiveColumnSizes(2, 3, 3, 3, 3), + new ResponsiveColumnSizes(3, 3, 3, 3, 3), // Format column new ResponsiveColumnSizes(2, 2, 2, 2, 2), // Actions column - new ResponsiveColumnSizes(6, 5, 4, 3, 3), + new ResponsiveColumnSizes(2, 2, 2, 2, 2), + // Store synchronization column + new ResponsiveColumnSizes(2, 2, 2, 2, 2), ]); } diff --git a/src/app/item-page/edit-item-page/item-bitstreams/item-edit-bitstream-bundle/item-edit-bitstream-bundle.component.html b/src/app/item-page/edit-item-page/item-bitstreams/item-edit-bitstream-bundle/item-edit-bitstream-bundle.component.html index f6f2e4579e1..1c85da844c8 100644 --- a/src/app/item-page/edit-item-page/item-bitstreams/item-edit-bitstream-bundle/item-edit-bitstream-bundle.component.html +++ b/src/app/item-page/edit-item-page/item-bitstreams/item-edit-bitstream-bundle/item-edit-bitstream-bundle.component.html @@ -15,18 +15,21 @@ [attr.aria-label]="'item.edit.bitstreams.bundle.table.aria-label' | translate: { bundle: bundleName } "> - + {{'item.edit.bitstreams.headers.name' | translate}} - + {{'item.edit.bitstreams.headers.description' | translate}} - + {{'item.edit.bitstreams.headers.format' | translate}} - + {{'item.edit.bitstreams.headers.actions' | translate}} + + {{'item.edit.bitstreams.headers.synchronized' | translate}} + @@ -72,13 +75,14 @@ + @for (entry of (tableEntries$ | async); track entry) { @if (updates[entry.id]; as update) { + scope="row" id="{{ entry.nameStripped }}" headers="{{ bundleName }} name-{{ sanitizedBundleName }}">
@@ -86,15 +90,15 @@ {{ entry.name }} + headers="{{ entry.nameStripped }} {{ bundleName }} description-{{ sanitizedBundleName }}"> {{ entry.description }} + headers="{{ entry.nameStripped }} {{ bundleName }} format-{{ sanitizedBundleName }}"> {{ (entry.format | async)?.shortDescription }} + headers="{{ entry.nameStripped }} {{ bundleName }} actions-{{ sanitizedBundleName }}">
+ +
+ + + + | +
+
+ } } @@ -136,3 +151,89 @@ } + + +
+ + + +
+ {{ 'item.edit.bitstreams.checksum.popover.warning.header' | translate }} +
+
+ {{ 'item.edit.bitstreams.checksum.popover.warning.body' | translate }} +
+
+ + +
+ {{ 'item.edit.bitstreams.checksum.popover.info.body' | translate }} +
+
+ + +
+
+
+ {{'item.edit.bitstreams.checksum.database' | translate}} +
+
+ {{'item.edit.bitstreams.checksum.algorithm' | translate}} {{bitstreamChecksum?.databaseChecksum?.checkSumAlgorithm}} +
+
+ {{'item.edit.bitstreams.checksum.value' | translate}} {{ bitstreamChecksum?.databaseChecksum?.value }} +
+
+
+
+ {{'item.edit.bitstreams.checksum.active-store' | translate}} +
+
+ {{'item.edit.bitstreams.checksum.algorithm' | translate}} {{bitstreamChecksum?.activeStore?.checkSumAlgorithm}} +
+
+ {{'item.edit.bitstreams.checksum.value' | translate}} {{ bitstreamChecksum?.activeStore?.value }} +
+
+ @if (isBitstreamSynchronized(entry)) { +
+
+ {{'item.edit.bitstreams.checksum.sync-store' | translate}} +
+
+ {{'item.edit.bitstreams.checksum.algorithm' | translate}} {{bitstreamChecksum?.synchronizedStore?.checkSumAlgorithm}} +
+
+ {{'item.edit.bitstreams.checksum.value' | translate}} {{ bitstreamChecksum?.synchronizedStore?.value }} +
+
+ } +
+
diff --git a/src/app/item-page/edit-item-page/item-bitstreams/item-edit-bitstream-bundle/item-edit-bitstream-bundle.component.spec.ts b/src/app/item-page/edit-item-page/item-bitstreams/item-edit-bitstream-bundle/item-edit-bitstream-bundle.component.spec.ts index ff0567eb9f2..c156105093f 100644 --- a/src/app/item-page/edit-item-page/item-bitstreams/item-edit-bitstream-bundle/item-edit-bitstream-bundle.component.spec.ts +++ b/src/app/item-page/edit-item-page/item-bitstreams/item-edit-bitstream-bundle/item-edit-bitstream-bundle.component.spec.ts @@ -14,12 +14,18 @@ import { Subject, } from 'rxjs'; +import { BitstreamChecksumDataService } from '../../../../core/bitstream-checksum-data.service'; import { BundleDataService } from '../../../../core/data/bundle-data.service'; import { FieldChangeType } from '../../../../core/data/object-updates/field-change-type.model'; import { FieldUpdate } from '../../../../core/data/object-updates/field-update.model'; import { ObjectUpdatesService } from '../../../../core/data/object-updates/object-updates.service'; import { RequestService } from '../../../../core/data/request.service'; import { PaginationService } from '../../../../core/pagination/pagination.service'; +import { + Bitstream, + SYNCHRONIZED_STORES_NUMBER, +} from '../../../../core/shared/bitstream.model'; +import { BitstreamChecksum } from '../../../../core/shared/bitstream-checksum.model'; import { Bundle } from '../../../../core/shared/bundle.model'; import { Item } from '../../../../core/shared/item.model'; import { getMockRequestService } from '../../../../shared/mocks/request.service.mock'; @@ -45,12 +51,35 @@ describe('ItemEditBitstreamBundleComponent', () => { let viewContainerRef: ViewContainerRef; const columnSizes = new ResponsiveTableSizes([ - new ResponsiveColumnSizes(2, 2, 3, 4, 4), - new ResponsiveColumnSizes(2, 3, 3, 3, 3), + new ResponsiveColumnSizes(3, 3, 3, 3, 3), + new ResponsiveColumnSizes(3, 3, 3, 3, 3), + new ResponsiveColumnSizes(2, 2, 2, 2, 2), + new ResponsiveColumnSizes(2, 2, 2, 2, 2), new ResponsiveColumnSizes(2, 2, 2, 2, 2), - new ResponsiveColumnSizes(6, 5, 4, 3, 3), ]); + + const md5 = (value: string) => ({ checkSumAlgorithm: 'MD5', value }); + const checksumAllEqual = Object.assign(new BitstreamChecksum(), { + databaseChecksum: md5('abc'), + activeStore: md5('abc'), + synchronizedStore: md5('abc'), + }); + + const tableEntry = (storeNumber: number, checksumHref?: string): BitstreamTableEntry => ({ + bitstream: Object.assign(new Bitstream(), { + uuid: 'bitstream-1', + storeNumber, + _links: checksumHref ? { checksum: { href: checksumHref } } : {}, + }), + id: 'bitstream-1', + name: 'file.txt', + nameStripped: 'file.txt', + description: '', + format: of(null), + downloadUrl: 'download-url', + } as any); + const item = Object.assign(new Item(), { id: 'item-1', uuid: 'item-1', @@ -71,6 +100,7 @@ describe('ItemEditBitstreamBundleComponent', () => { let objectUpdatesService: any; let itemBitstreamsService: ItemBitstreamsServiceStub; + let bitstreamChecksumService: jasmine.SpyObj; beforeEach(waitForAsync(() => { objectUpdatesService = jasmine.createSpyObj('objectUpdatesService', { @@ -79,6 +109,8 @@ describe('ItemEditBitstreamBundleComponent', () => { }); itemBitstreamsService = getItemBitstreamsServiceStub(); + bitstreamChecksumService = jasmine.createSpyObj('bitstreamChecksumService', ['findByHref']); + bitstreamChecksumService.findByHref.and.returnValue(createSuccessfulRemoteDataObject$(checksumAllEqual)); TestBed.configureTestingModule({ imports: [TranslateModule.forRoot(), ItemEditBitstreamBundleComponent], @@ -88,6 +120,7 @@ describe('ItemEditBitstreamBundleComponent', () => { { provide: PaginationService, useValue: new PaginationServiceStub() }, { provide: RequestService, useValue: getMockRequestService() }, { provide: ItemBitstreamsService, useValue: itemBitstreamsService }, + { provide: BitstreamChecksumDataService, useValue: bitstreamChecksumService }, ], schemas: [ NO_ERRORS_SCHEMA, @@ -356,4 +389,53 @@ describe('ItemEditBitstreamBundleComponent', () => { expect(itemBitstreamsService.cancelSelection).not.toHaveBeenCalled(); }); }); + + describe('bitstream checksum column', () => { + + it('should report a bitstream stored in both stores as synchronized', () => { + expect(comp.isBitstreamSynchronized(tableEntry(SYNCHRONIZED_STORES_NUMBER))).toBeTrue(); + expect(comp.isBitstreamSynchronized(tableEntry(0))).toBeFalse(); + }); + + it('should treat checksums with the same value but a different algorithm as unequal', () => { + expect(comp.compareChecksums(md5('abc'), md5('abc'))).toBeTrue(); + expect(comp.compareChecksums(md5('abc'), { checkSumAlgorithm: 'SHA-256', value: 'abc' })).toBeFalse(); + }); + + it('should compare the synchronized store as well, but only for a synchronized bitstream', () => { + const syncStoreDiffers = Object.assign(new BitstreamChecksum(), { + databaseChecksum: md5('abc'), + activeStore: md5('abc'), + synchronizedStore: md5('zzz'), + }); + + expect(comp.checksumsAreEqual(syncStoreDiffers, tableEntry(SYNCHRONIZED_STORES_NUMBER))).toBeFalse(); + expect(comp.checksumsAreEqual(syncStoreDiffers, tableEntry(0))).toBeTrue(); + }); + + it('should request the checksum of the row it was given, not of the selected row', () => { + const entry = tableEntry(SYNCHRONIZED_STORES_NUMBER, 'https://rest/api/core/bitstreams/bitstream-1/checksum'); + + comp.computeChecksum(entry); + + expect(bitstreamChecksumService.findByHref) + .toHaveBeenCalledWith('https://rest/api/core/bitstreams/bitstream-1/checksum'); + expect(itemBitstreamsService.getSelectedBitstream).not.toHaveBeenCalled(); + + let emitted: BitstreamChecksum = null; + comp.checkSum$.subscribe((value) => emitted = value); + + expect(emitted).toBe(checksumAllEqual); + expect(comp.computedChecksum).toBeTrue(); + expect(comp.loading).toBeFalse(); + }); + + it('should do nothing when the row has no checksum link', () => { + const entry = tableEntry(0); + + expect(() => comp.computeChecksum(entry)).not.toThrow(); + expect(bitstreamChecksumService.findByHref).not.toHaveBeenCalled(); + expect(comp.loading).toBeFalse(); + }); + }); }); diff --git a/src/app/item-page/edit-item-page/item-bitstreams/item-edit-bitstream-bundle/item-edit-bitstream-bundle.component.ts b/src/app/item-page/edit-item-page/item-bitstreams/item-edit-bitstream-bundle/item-edit-bitstream-bundle.component.ts index 346113af7e0..8057418a5b3 100644 --- a/src/app/item-page/edit-item-page/item-bitstreams/item-edit-bitstream-bundle/item-edit-bitstream-bundle.component.ts +++ b/src/app/item-page/edit-item-page/item-bitstreams/item-edit-bitstream-bundle/item-edit-bitstream-bundle.component.ts @@ -15,6 +15,7 @@ import { import { RouterLink } from '@angular/router'; import { NgbDropdownModule, + NgbPopover, NgbTooltip, } from '@ng-bootstrap/ng-bootstrap'; import { TranslateModule } from '@ngx-translate/core'; @@ -33,8 +34,12 @@ import { } from 'rxjs/operators'; import { PaginatedList } from 'src/app/core/data/paginated-list.model'; import { RemoteData } from 'src/app/core/data/remote-data'; -import { Bitstream } from 'src/app/core/shared/bitstream.model'; +import { + Bitstream, + SYNCHRONIZED_STORES_NUMBER, +} from 'src/app/core/shared/bitstream.model'; +import { BitstreamChecksumDataService } from '../../../../core/bitstream-checksum-data.service'; import { DSONameService } from '../../../../core/breadcrumbs/dso-name.service'; import { BundleDataService } from '../../../../core/data/bundle-data.service'; import { FieldChangeType } from '../../../../core/data/object-updates/field-change-type.model'; @@ -43,10 +48,16 @@ import { FieldUpdates } from '../../../../core/data/object-updates/field-updates import { ObjectUpdatesService } from '../../../../core/data/object-updates/object-updates.service'; import { RequestService } from '../../../../core/data/request.service'; import { PaginationService } from '../../../../core/pagination/pagination.service'; +import { + BitstreamChecksum, + CheckSum, +} from '../../../../core/shared/bitstream-checksum.model'; import { Bundle } from '../../../../core/shared/bundle.model'; import { Item } from '../../../../core/shared/item.model'; import { getAllSucceededRemoteData, + getFirstSucceededRemoteData, + getRemoteDataPayload, paginatedListToArray, } from '../../../../core/shared/operators'; import { BtnDisabledDirective } from '../../../../shared/btn-disabled.directive'; @@ -61,6 +72,7 @@ import { ResponsiveTableSizes } from '../../../../shared/responsive-table-sizes/ import { PaginatedSearchOptions } from '../../../../shared/search/models/paginated-search-options.model'; import { BrowserOnlyPipe } from '../../../../shared/utils/browser-only.pipe'; import { followLink } from '../../../../shared/utils/follow-link-config.model'; +import { VarDirective } from '../../../../shared/utils/var.directive'; import { getItemPageRoute } from '../../../item-page-routing-paths'; import { BitstreamTableEntry, @@ -81,10 +93,12 @@ import { CdkDropList, CommonModule, NgbDropdownModule, + NgbPopover, NgbTooltip, PaginationComponent, RouterLink, TranslateModule, + VarDirective, ], }) /** @@ -146,6 +160,12 @@ export class ItemEditBitstreamBundleComponent implements OnInit, OnDestroy { */ bundleName: string; + /** + * Sanitized bundle name (whitespace removed) for use in HTML element IDs. + * HTML IDs cannot contain spaces, so this ensures valid id/headers/aria-labelledby tokens. + */ + sanitizedBundleName: string; + /** * The number of bitstreams in the bundle */ @@ -191,6 +211,25 @@ export class ItemEditBitstreamBundleComponent implements OnInit, OnDestroy { */ subscriptions: Subscription[] = []; + /** + * True on mouseover, false otherwise + */ + showChecksumValues = false; + + /** + * Object containing all checksums + */ + checkSum$: Observable; + + /** + * Compute checksum - the whole file must be downloaded to compute the checksum + */ + computedChecksum = false; + + /** + * True if the bitstream is being downloaded and the checksum is being computed + */ + loading = false; constructor( protected viewContainerRef: ViewContainerRef, @@ -200,6 +239,7 @@ export class ItemEditBitstreamBundleComponent implements OnInit, OnDestroy { protected paginationService: PaginationService, protected requestService: RequestService, protected itemBitstreamsService: ItemBitstreamsService, + protected bitstreamChecksumDataService: BitstreamChecksumDataService, ) { } @@ -208,6 +248,7 @@ export class ItemEditBitstreamBundleComponent implements OnInit, OnDestroy { this.viewContainerRef.createEmbeddedView(this.bundleView); this.itemPageRoute = getItemPageRoute(this.item); this.bundleName = this.dsoNameService.getName(this.bundle); + this.sanitizedBundleName = (this.bundleName ?? '').replace(/\s+/g, ''); this.bundleUrl = this.bundle.self; this.initializePagination(); @@ -587,4 +628,70 @@ export class ItemEditBitstreamBundleComponent implements OnInit, OnDestroy { this.paginationComponent.doPageChange(page + 1); } + /** + * Compare if two checksums are equal + * + * @param checksum1 e.g. DB checksum + * @param checksum2 e.g. Active store checksum (local or S3) + */ + compareChecksums(checksum1: CheckSum, checksum2: CheckSum): boolean { + return checksum1?.value === checksum2?.value && checksum1?.checkSumAlgorithm === checksum2?.checkSumAlgorithm; + } + + /** + * Compare if all checksums are equal (DB, Active store, Synchronized store) + * + * @param bitstreamChecksum which contains all checksums + * @param entry the table row the checksum belongs to + */ + checksumsAreEqual(bitstreamChecksum: BitstreamChecksum, entry: BitstreamTableEntry): boolean { + if (hasNoValue(bitstreamChecksum)) { + return false; + } + + if (this.isBitstreamSynchronized(entry)) { + // Compare DB and Active store checksums + // Compare DB and Synchronized and Active store checksums + return this.compareChecksums(bitstreamChecksum.databaseChecksum, bitstreamChecksum.activeStore) && + this.compareChecksums(bitstreamChecksum.synchronizedStore, bitstreamChecksum.activeStore); + } + // Compare DB and Active store checksums + return this.compareChecksums(bitstreamChecksum.databaseChecksum, bitstreamChecksum.activeStore); + } + + /** + * Check if the bitstream of this row is stored in both stores (S3 and local) + * + * @param entry the table row to evaluate + */ + isBitstreamSynchronized(entry: BitstreamTableEntry): boolean { + return entry?.bitstream?.storeNumber === SYNCHRONIZED_STORES_NUMBER; + } + + /** + * Fetch the checksums of the bitstream in the provided row. The backend downloads the whole file to + * compute the store checksums, so this only runs when the admin asks for it. + * + * @param entry the table row whose checksum should be computed + */ + computeChecksum(entry: BitstreamTableEntry) { + const href = entry?.bitstream?._links?.checksum?.href; + if (hasNoValue(href)) { + return; + } + + this.loading = true; + // Send request to get bitstream checksum + this.checkSum$ = this.bitstreamChecksumDataService.findByHref(href) + .pipe( + getFirstSucceededRemoteData(), + getRemoteDataPayload(), + map(value => { + this.computedChecksum = true; + this.loading = false; + return value; + }), + ); + } + }