From 7ed1c36055453fd84fe45b543b3b97c0266326a1 Mon Sep 17 00:00:00 2001 From: asarode9 Date: Tue, 7 Apr 2026 05:16:36 -0400 Subject: [PATCH 1/8] Implemented file delete feature --- packages/api-gateway/src/app.module.ts | 2 + .../api-gateway/src/models/file_delete.dto.ts | 49 ++++++++++ .../file_delete/file_delete.controller.ts | 74 +++++++++++++++ .../modules/file_delete/file_delete.module.ts | 72 +++++++++++++++ .../modules/file_transfer/azure_handler.ts | 40 ++++++++ .../file_transfer/file_transfer.controller.ts | 91 ++++++++++++++++++- .../file_transfer/file_transfer.module.ts | 12 +++ .../src/modules/file_transfer/s3_handler.ts | 39 ++++++++ packages/proto/definitions/file.proto | 13 +++ packages/proto/src/gen/file.ts | 23 ++++- 10 files changed, 413 insertions(+), 2 deletions(-) create mode 100644 packages/api-gateway/src/models/file_delete.dto.ts create mode 100644 packages/api-gateway/src/modules/file_delete/file_delete.controller.ts create mode 100644 packages/api-gateway/src/modules/file_delete/file_delete.module.ts diff --git a/packages/api-gateway/src/app.module.ts b/packages/api-gateway/src/app.module.ts index d475cce1..afd72ce7 100644 --- a/packages/api-gateway/src/app.module.ts +++ b/packages/api-gateway/src/app.module.ts @@ -11,6 +11,7 @@ import { FileBucketModule } from './modules/file_bucket/file_bucket.module'; import { FileConfigModule } from './modules/file_config/file_config.module'; import { AnalyticsModule } from './modules/analytics/analytics.module'; import { AnalyticsConfigModule } from './modules/analytics_config/analytics_config.module'; +import { FileDeleteModule } from './modules/file_delete/file_delete.module'; @Module({ imports: [ @@ -24,6 +25,7 @@ import { AnalyticsConfigModule } from './modules/analytics_config/analytics_conf FileUploadModule, FileBucketModule, FileConfigModule, + FileDeleteModule, AnalyticsModule, AnalyticsConfigModule, ], diff --git a/packages/api-gateway/src/models/file_delete.dto.ts b/packages/api-gateway/src/models/file_delete.dto.ts new file mode 100644 index 00000000..3f0a10d3 --- /dev/null +++ b/packages/api-gateway/src/models/file_delete.dto.ts @@ -0,0 +1,49 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsArray, IsInt, IsNotEmpty, IsString, Min } from 'class-validator'; +import { FileProto } from 'juno-proto'; + +export class DeleteFilesModel { + @ApiProperty({ + type: 'string', + description: 'Name of the bucket containing the files', + }) + @IsNotEmpty() + @IsString() + bucketName: string; + + @ApiProperty({ + type: 'number', + description: 'Configuration ID for the bucket', + }) + @IsInt() + @Min(0) + @IsNotEmpty() + configId: number; + + @ApiProperty({ + type: [String], + description: 'List of file names to delete', + }) + @IsArray() + @IsNotEmpty() + fileNames: string[]; +} + +export class DeleteFilesResponse { + @ApiProperty({ + type: 'string', + description: 'The name of the bucket from which files were deleted', + }) + bucketName: string; + + @ApiProperty({ + type: [String], + description: 'List of file names that were deleted', + }) + fileNames: string[]; + + constructor(res: FileProto.DeleteFilesResponse) { + this.bucketName = res.bucketName; + this.fileNames = res.fileNames; + } +} diff --git a/packages/api-gateway/src/modules/file_delete/file_delete.controller.ts b/packages/api-gateway/src/modules/file_delete/file_delete.controller.ts new file mode 100644 index 00000000..c00b25ee --- /dev/null +++ b/packages/api-gateway/src/modules/file_delete/file_delete.controller.ts @@ -0,0 +1,74 @@ +import { + Body, + Controller, + Delete, + HttpStatus, + Inject, + OnModuleInit, +} from '@nestjs/common'; +import { ClientGrpc } from '@nestjs/microservices'; +import { lastValueFrom } from 'rxjs'; +import { AuthCommonProto, FileProto } from 'juno-proto'; +import { + ApiBearerAuth, + ApiOperation, + ApiResponse, + ApiTags, +} from '@nestjs/swagger'; +import { + DeleteFilesModel, + DeleteFilesResponse, +} from 'src/models/file_delete.dto'; +import { ApiKey } from 'src/decorators/api_key.decorator'; + +const { FILE_SERVICE_NAME } = FileProto; + +@ApiBearerAuth('API_Key') +@ApiTags('file_delete') +@Controller('file') +export class FileDeleteController implements OnModuleInit { + private fileService: FileProto.FileServiceClient; + + constructor( + @Inject(FILE_SERVICE_NAME) + private fileClient: ClientGrpc, + ) {} + + onModuleInit() { + this.fileService = + this.fileClient.getService( + FILE_SERVICE_NAME, + ); + } + + @Delete('delete') + @ApiOperation({ summary: 'Delete files from a bucket' }) + @ApiResponse({ + status: HttpStatus.BAD_REQUEST, + description: 'Parameters are invalid', + }) + @ApiResponse({ + status: HttpStatus.NOT_FOUND, + description: 'Bucket or file not found', + }) + @ApiResponse({ + status: HttpStatus.OK, + description: 'Returns the bucket name and list of deleted file names', + type: DeleteFilesResponse, + }) + async deleteFiles( + @ApiKey() apiKey: AuthCommonProto.ApiKey, + @Body() params: DeleteFilesModel, + ): Promise { + const res = await lastValueFrom( + this.fileService.deleteFiles({ + bucketName: params.bucketName, + configId: params.configId, + configEnv: apiKey.environment, + fileNames: params.fileNames, + }), + ); + + return new DeleteFilesResponse(res); + } +} diff --git a/packages/api-gateway/src/modules/file_delete/file_delete.module.ts b/packages/api-gateway/src/modules/file_delete/file_delete.module.ts new file mode 100644 index 00000000..2a443a53 --- /dev/null +++ b/packages/api-gateway/src/modules/file_delete/file_delete.module.ts @@ -0,0 +1,72 @@ +import { + Module, + NestModule, + RequestMethod, + MiddlewareConsumer, +} from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; +import { join } from 'path'; +import { FileDeleteController } from './file_delete.controller'; +import { ApiKeyMiddleware } from 'src/middleware/api_key.middleware'; + +import { ClientsModule, Transport } from '@nestjs/microservices'; +import { + ApiKeyProtoFile, + FileProto, + FileProtoFile, + JwtProtoFile, +} from 'juno-proto'; +import { FILE_SERVICE_NAME } from 'juno-proto/dist/gen/file'; +import { + API_KEY_SERVICE_NAME, + JUNO_API_KEY_PACKAGE_NAME, +} from 'juno-proto/dist/gen/api_key'; +import { + JWT_SERVICE_NAME, + JUNO_JWT_PACKAGE_NAME, +} from 'juno-proto/dist/gen/jwt'; + +@Module({ + imports: [ + ConfigModule.forRoot({ + envFilePath: join(__dirname, '../../../../../.env.local'), + }), + ClientsModule.register([ + { + name: API_KEY_SERVICE_NAME, + transport: Transport.GRPC, + options: { + url: process.env.AUTH_SERVICE_ADDR, + package: JUNO_API_KEY_PACKAGE_NAME, + protoPath: ApiKeyProtoFile, + }, + }, + { + name: JWT_SERVICE_NAME, + transport: Transport.GRPC, + options: { + url: process.env.AUTH_SERVICE_ADDR, + package: JUNO_JWT_PACKAGE_NAME, + protoPath: JwtProtoFile, + }, + }, + { + name: FILE_SERVICE_NAME, + transport: Transport.GRPC, + options: { + url: process.env.FILE_SERVICE_ADDR, + package: FileProto.JUNO_FILE_SERVICE_FILE_PACKAGE_NAME, + protoPath: FileProtoFile, + }, + }, + ]), + ], + controllers: [FileDeleteController], +}) +export class FileDeleteModule implements NestModule { + configure(consumer: MiddlewareConsumer) { + consumer + .apply(ApiKeyMiddleware) + .forRoutes({ path: '/file/delete', method: RequestMethod.DELETE }); + } +} diff --git a/packages/file-service/src/modules/file_transfer/azure_handler.ts b/packages/file-service/src/modules/file_transfer/azure_handler.ts index 072cf91d..332e7073 100644 --- a/packages/file-service/src/modules/file_transfer/azure_handler.ts +++ b/packages/file-service/src/modules/file_transfer/azure_handler.ts @@ -4,6 +4,7 @@ import { status } from '@grpc/grpc-js'; import { lastValueFrom } from 'rxjs'; import { BlobSASPermissions, + BlobServiceClient, generateBlobSASQueryParameters, StorageSharedKeyCredential, } from '@azure/storage-blob'; @@ -14,6 +15,26 @@ export class AzureFileHandler { private provider: FileProviderProto.FileProvider, ) {} + async getBlobServiceClient(): Promise { + const accessKeyPayload = JSON.parse(this.provider.accessKey); + if (!accessKeyPayload.accountName || !accessKeyPayload.accountKey) { + throw new RpcException({ + code: status.FAILED_PRECONDITION, + message: 'Invalid access key payload', + }); + } + const account = accessKeyPayload.accountName; + const accountKey = accessKeyPayload.accountKey; + const sharedKeyCredential = new StorageSharedKeyCredential( + account, + accountKey, + ); + return new BlobServiceClient( + `https://${account}.blob.core.windows.net`, + sharedKeyCredential, + ); + } + async getSharedKeyCredential(): Promise { const accessKeyPayload = JSON.parse(this.provider.accessKey); if (!accessKeyPayload.accountName || !accessKeyPayload.accountKey) { @@ -28,6 +49,25 @@ export class AzureFileHandler { return new StorageSharedKeyCredential(account, accountKey); } + async deleteFiles(request: FileProto.DeleteFilesRequest): Promise { + try { + const blobServiceClient = await this.getBlobServiceClient(); + const containerClient = blobServiceClient.getContainerClient( + `${request.bucketName}-${request.configId}-${request.configEnv}`, + ); + await Promise.all( + request.fileNames.map((name) => + containerClient.getBlobClient(name).deleteIfExists(), + ), + ); + } catch (err) { + throw new RpcException({ + code: status.FAILED_PRECONDITION, + message: `Failed to delete files from Azure: ${err}`, + }); + } + } + async downloadFile( request: FileProto.DownloadFileRequest, ): Promise { diff --git a/packages/file-service/src/modules/file_transfer/file_transfer.controller.ts b/packages/file-service/src/modules/file_transfer/file_transfer.controller.ts index c230b50a..0be288e6 100644 --- a/packages/file-service/src/modules/file_transfer/file_transfer.controller.ts +++ b/packages/file-service/src/modules/file_transfer/file_transfer.controller.ts @@ -1,7 +1,7 @@ import { status } from '@grpc/grpc-js'; import { Controller, Inject } from '@nestjs/common'; import { ClientGrpc, RpcException } from '@nestjs/microservices'; -import { FileProto, FileProviderProto } from 'juno-proto'; +import { FileBucketProto, FileProto, FileProviderProto } from 'juno-proto'; import { FileServiceController } from 'juno-proto/dist/gen/file'; import { lastValueFrom } from 'rxjs'; import { AzureFileHandler } from './azure_handler'; @@ -9,17 +9,21 @@ import { S3FileHandler } from './s3_handler'; const { FILE_DB_SERVICE_NAME } = FileProto; const { FILE_PROVIDER_DB_SERVICE_NAME } = FileProviderProto; +const { BUCKET_DB_SERVICE_NAME } = FileBucketProto; @Controller() @FileProto.FileServiceControllerMethods() export class FileTransferController implements FileServiceController { private fileDBService: FileProto.FileDbServiceClient; private fileProviderDbService: FileProviderProto.FileProviderDbServiceClient; + private fileBucketDBService: FileBucketProto.BucketDbServiceClient; constructor( @Inject(FILE_DB_SERVICE_NAME) private fileDBClient: ClientGrpc, @Inject(FILE_PROVIDER_DB_SERVICE_NAME) private fileProviderClient: ClientGrpc, + @Inject(BUCKET_DB_SERVICE_NAME) + private fileBucketClient: ClientGrpc, ) {} onModuleInit() { @@ -31,6 +35,91 @@ export class FileTransferController implements FileServiceController { this.fileProviderClient.getService( FileProviderProto.FILE_PROVIDER_DB_SERVICE_NAME, ); + this.fileBucketDBService = + this.fileBucketClient.getService( + BUCKET_DB_SERVICE_NAME, + ); + } + + async deleteFiles( + request: FileProto.DeleteFilesRequest, + ): Promise { + if ( + !request || + !request.bucketName || + request.configId == undefined || + !request.configEnv || + !request.fileNames?.length + ) { + throw new RpcException({ + code: status.INVALID_ARGUMENT, + message: + 'bucketName, configId, configEnv, and fileNames must all be provided', + }); + } + + try { + const bucket = await lastValueFrom( + this.fileBucketDBService.getBucket({ + name: request.bucketName, + configId: request.configId, + configEnv: request.configEnv, + }), + ); + + const provider = await lastValueFrom( + this.fileProviderDbService.getProvider({ + providerName: bucket.fileProviderName, + }), + ); + + switch (provider.providerType) { + case FileProviderProto.ProviderType.S3: { + const handler = new S3FileHandler(this.fileDBService, provider); + await handler.deleteFiles(request); + break; + } + case FileProviderProto.ProviderType.AZURE: { + const handler = new AzureFileHandler(this.fileDBService, provider); + await handler.deleteFiles(request); + break; + } + default: { + throw new RpcException({ + code: status.FAILED_PRECONDITION, + message: 'Provider type not supported', + }); + } + } + + await Promise.all( + request.fileNames.map((fileName) => + lastValueFrom( + this.fileDBService.deleteFile({ + fileId: { + bucketName: request.bucketName, + configId: request.configId, + path: fileName, + configEnv: request.configEnv, + }, + }), + ), + ), + ); + + return { + bucketName: request.bucketName, + fileNames: request.fileNames, + }; + } catch (err) { + if (err instanceof RpcException) { + throw err; + } + throw new RpcException({ + code: err.code ?? status.INTERNAL, + message: `Unknown error occurred: ${err}`, + }); + } } async downloadFile( diff --git a/packages/file-service/src/modules/file_transfer/file_transfer.module.ts b/packages/file-service/src/modules/file_transfer/file_transfer.module.ts index 23b39b48..6fef19f7 100644 --- a/packages/file-service/src/modules/file_transfer/file_transfer.module.ts +++ b/packages/file-service/src/modules/file_transfer/file_transfer.module.ts @@ -2,6 +2,8 @@ import { Module } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; import { ClientsModule, Transport } from '@nestjs/microservices'; import { + FileBucketProto, + FileBucketProtoFile, FileProto, FileProtoFile, FileProviderProto, @@ -12,6 +14,7 @@ import { FileTransferController } from './file_transfer.controller'; const { FILE_DB_SERVICE_NAME, JUNO_FILE_SERVICE_FILE_PACKAGE_NAME } = FileProto; const { FILE_PROVIDER_DB_SERVICE_NAME } = FileProviderProto; +const { BUCKET_DB_SERVICE_NAME } = FileBucketProto; @Module({ imports: [ @@ -37,6 +40,15 @@ const { FILE_PROVIDER_DB_SERVICE_NAME } = FileProviderProto; protoPath: FileProviderProtoFile, }, }, + { + name: BUCKET_DB_SERVICE_NAME, + transport: Transport.GRPC, + options: { + url: process.env.DB_SERVICE_ADDR, + package: FileBucketProto.JUNO_FILE_SERVICE_BUCKET_PACKAGE_NAME, + protoPath: FileBucketProtoFile, + }, + }, ]), ], controllers: [FileTransferController], diff --git a/packages/file-service/src/modules/file_transfer/s3_handler.ts b/packages/file-service/src/modules/file_transfer/s3_handler.ts index fa6d6a85..73d9b7ab 100644 --- a/packages/file-service/src/modules/file_transfer/s3_handler.ts +++ b/packages/file-service/src/modules/file_transfer/s3_handler.ts @@ -2,6 +2,8 @@ import { S3Client, GetObjectCommand, PutObjectCommand, + DeleteObjectsCommand, + ListObjectVersionsCommand, } from '@aws-sdk/client-s3'; import { FileProto, FileProviderProto } from 'juno-proto'; import { RpcException } from '@nestjs/microservices'; @@ -76,6 +78,43 @@ export class S3FileHandler { } } + async deleteFiles(request: FileProto.DeleteFilesRequest): Promise { + try { + const s3Client = await this.getS3Client('us-east-005'); + const bucket = `${request.bucketName}-${request.configId}-${request.configEnv}`; + + const objectsToDelete: { Key: string; VersionId?: string }[] = []; + + for (const name of request.fileNames) { + const listCommand = new ListObjectVersionsCommand({ + Bucket: bucket, + Prefix: name, + }); + const versions = await s3Client.send(listCommand); + + for (const version of versions.Versions ?? []) { + objectsToDelete.push({ Key: version.Key!, VersionId: version.VersionId }); + } + for (const marker of versions.DeleteMarkers ?? []) { + objectsToDelete.push({ Key: marker.Key!, VersionId: marker.VersionId }); + } + } + + if (objectsToDelete.length > 0) { + const deleteCommand = new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: objectsToDelete }, + }); + await s3Client.send(deleteCommand); + } + } catch (err) { + throw new RpcException({ + code: status.FAILED_PRECONDITION, + message: `Failed to delete files from S3: ${err}`, + }); + } + } + async uploadFile( request: FileProto.UploadFileRequest, ): Promise { diff --git a/packages/proto/definitions/file.proto b/packages/proto/definitions/file.proto index a865c812..ee20210b 100644 --- a/packages/proto/definitions/file.proto +++ b/packages/proto/definitions/file.proto @@ -14,6 +14,7 @@ service FileDbService { service FileService { rpc uploadFile(UploadFileRequest) returns (UploadFileResponse); rpc downloadFile(DownloadFileRequest) returns (DownloadFileResponse); + rpc deleteFiles(DeleteFilesRequest) returns (DeleteFilesResponse); } message File { @@ -56,3 +57,15 @@ message DownloadFileRequest { } message DownloadFileResponse { string url = 1; } + +message DeleteFilesRequest { + string bucketName = 1; + int32 configId = 2; + string configEnv = 3; + repeated string fileNames = 4; +} + +message DeleteFilesResponse { + string bucketName = 1; + repeated string fileNames = 2; +} diff --git a/packages/proto/src/gen/file.ts b/packages/proto/src/gen/file.ts index b9c15314..a849fcd9 100644 --- a/packages/proto/src/gen/file.ts +++ b/packages/proto/src/gen/file.ts @@ -60,6 +60,18 @@ export interface DownloadFileResponse { url: string; } +export interface DeleteFilesRequest { + bucketName: string; + configId: number; + configEnv: string; + fileNames: string[]; +} + +export interface DeleteFilesResponse { + bucketName: string; + fileNames: string[]; +} + export const JUNO_FILE_SERVICE_FILE_PACKAGE_NAME = 'juno.file_service.file'; export interface FileDbServiceClient { @@ -128,6 +140,8 @@ export interface FileServiceClient { uploadFile(request: UploadFileRequest): Observable; downloadFile(request: DownloadFileRequest): Observable; + + deleteFiles(request: DeleteFilesRequest): Observable; } export interface FileServiceController { @@ -144,11 +158,18 @@ export interface FileServiceController { | Promise | Observable | DownloadFileResponse; + + deleteFiles( + request: DeleteFilesRequest, + ): + | Promise + | Observable + | DeleteFilesResponse; } export function FileServiceControllerMethods() { return function (constructor: Function) { - const grpcMethods: string[] = ['uploadFile', 'downloadFile']; + const grpcMethods: string[] = ['uploadFile', 'downloadFile', 'deleteFiles']; for (const method of grpcMethods) { const descriptor: any = Reflect.getOwnPropertyDescriptor( constructor.prototype, From d9f03cb0f5123ac09b7ea2a56ae315958df52c80 Mon Sep 17 00:00:00 2001 From: asarode9 Date: Tue, 7 Apr 2026 05:16:44 -0400 Subject: [PATCH 2/8] implemented e2e tests --- .../api-gateway/test/file_delete.e2e-spec.ts | 211 ++++++++++++++ .../file-service/test/file_delete.e2e-spec.ts | 273 ++++++++++++++++++ 2 files changed, 484 insertions(+) create mode 100644 packages/api-gateway/test/file_delete.e2e-spec.ts create mode 100644 packages/file-service/test/file_delete.e2e-spec.ts diff --git a/packages/api-gateway/test/file_delete.e2e-spec.ts b/packages/api-gateway/test/file_delete.e2e-spec.ts new file mode 100644 index 00000000..98916a9b --- /dev/null +++ b/packages/api-gateway/test/file_delete.e2e-spec.ts @@ -0,0 +1,211 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { + ClassSerializerInterceptor, + INestApplication, + ValidationPipe, +} from '@nestjs/common'; +import { AppModule } from './../src/app.module'; +import { Reflector } from '@nestjs/core'; +import * as request from 'supertest'; +import { + ResetProtoFile, + FileProviderProtoFile, + FileBucketProtoFile, + FileProtoFile, + FileProviderProto, +} from 'juno-proto'; +import * as GRPC from '@grpc/grpc-js'; +import * as ProtoLoader from '@grpc/proto-loader'; +import { RpcExceptionFilter } from 'src/rpc_exception_filter'; + +let app: INestApplication; +const ADMIN_EMAIL = 'test-superadmin@test.com'; +const ADMIN_PASSWORD = 'test-password'; +let apiKey: string | undefined = undefined; + +async function APIKeyForProjectName(projectName: string): Promise { + const key = await request(app.getHttpServer()) + .post('/auth/key') + .set('X-User-Email', ADMIN_EMAIL) + .set('X-User-Password', ADMIN_PASSWORD) + .send({ + environment: 'prod', + project: { + name: projectName, + }, + }); + + return key.body['apiKey']; +} + +const bucketName = 'test-delete-gw-juno'; +const configId = 0; +const providerName = 'backblazeb2-delete-gw'; + +const accessKeyId = process.env.accessKeyId; +const secretAccessKey = process.env.secretAccessKey; +const baseURL = process.env.baseURL; + +beforeAll(async () => { + const proto = ProtoLoader.loadSync([ResetProtoFile]) as any; + const protoGRPC = GRPC.loadPackageDefinition(proto) as any; + const resetClient = new protoGRPC.juno.reset_db.DatabaseReset( + process.env.DB_SERVICE_ADDR, + GRPC.credentials.createInsecure(), + ); + await new Promise((resolve) => { + resetClient.resetDb({}, () => resolve(0)); + }); + + const providerProto = ProtoLoader.loadSync([FileProviderProtoFile]) as any; + const providerProtoGRPC = GRPC.loadPackageDefinition(providerProto) as any; + const providerClient = + new providerProtoGRPC.juno.file_service.provider.FileProviderDbService( + process.env.DB_SERVICE_ADDR, + GRPC.credentials.createInsecure(), + ); + + await new Promise((resolve) => { + providerClient.createProvider( + { + providerName, + accessKey: JSON.stringify({ accessKeyId, secretAccessKey }), + metadata: JSON.stringify({ endpoint: baseURL }), + bucket: [], + type: FileProviderProto.ProviderType.S3, + }, + () => resolve(0), + ); + }); + + const bucketProto = ProtoLoader.loadSync([FileBucketProtoFile]) as any; + const bucketProtoGRPC = GRPC.loadPackageDefinition(bucketProto) as any; + const bucketClient = + new bucketProtoGRPC.juno.file_service.bucket.BucketDbService( + process.env.DB_SERVICE_ADDR, + GRPC.credentials.createInsecure(), + ); + + await new Promise((resolve) => { + bucketClient.createBucket( + { + name: bucketName, + configId, + fileProviderName: providerName, + files: [], + configEnv: 'prod', + }, + () => resolve(0), + ); + }); +}); + +afterAll((done) => { + app.close(); + done(); +}); + +beforeEach(async () => { + const moduleFixture: TestingModule = await Test.createTestingModule({ + imports: [AppModule], + }).compile(); + + app = moduleFixture.createNestApplication(); + app.useGlobalPipes( + new ValidationPipe({ + transform: true, + }), + ); + app.useGlobalInterceptors(new ClassSerializerInterceptor(app.get(Reflector))); + app.useGlobalFilters(new RpcExceptionFilter()); + + await app.init(); + + if (!apiKey) { + apiKey = await APIKeyForProjectName('test-seed-project'); + } +}); + +describe('File Delete Routes', () => { + it('Rejects delete without auth key', () => { + return request(app.getHttpServer()) + .delete('/file/delete') + .send({ + bucketName, + configId, + fileNames: ['SomeFile'], + }) + .expect(401); + }); + + it('Rejects delete with missing bucketName', () => { + return request(app.getHttpServer()) + .delete('/file/delete') + .set('Authorization', 'Bearer ' + apiKey) + .send({ + bucketName: '', + configId, + fileNames: ['SomeFile'], + }) + .expect(400); + }); + + it('Rejects delete with negative configId', () => { + return request(app.getHttpServer()) + .delete('/file/delete') + .set('Authorization', 'Bearer ' + apiKey) + .send({ + bucketName, + configId: -1, + fileNames: ['SomeFile'], + }) + .expect(400); + }); + + it('Rejects delete with empty fileNames', () => { + return request(app.getHttpServer()) + .delete('/file/delete') + .set('Authorization', 'Bearer ' + apiKey) + .send({ + bucketName, + configId, + fileNames: [], + }) + .expect(400); + }); + + it('Successfully deletes a file', async () => { + // Upload the file first via the file service proto directly + const fileProto = ProtoLoader.loadSync([FileProtoFile]) as any; + const fileProtoGRPC = GRPC.loadPackageDefinition(fileProto) as any; + const fileClient = new fileProtoGRPC.juno.file_service.file.FileDbService( + process.env.DB_SERVICE_ADDR, + GRPC.credentials.createInsecure(), + ); + + await new Promise((resolve) => { + fileClient.createFile( + { + fileId: { + bucketName, + configId, + configEnv: 'prod', + path: 'FileToDeleteGW', + }, + metadata: '', + }, + () => resolve(0), + ); + }); + + return request(app.getHttpServer()) + .delete('/file/delete') + .set('Authorization', 'Bearer ' + apiKey) + .send({ + bucketName, + configId, + fileNames: ['FileToDeleteGW'], + }) + .expect(200); + }); +}); diff --git a/packages/file-service/test/file_delete.e2e-spec.ts b/packages/file-service/test/file_delete.e2e-spec.ts new file mode 100644 index 00000000..643d6788 --- /dev/null +++ b/packages/file-service/test/file_delete.e2e-spec.ts @@ -0,0 +1,273 @@ +import { INestMicroservice } from '@nestjs/common'; +import { MicroserviceOptions, Transport } from '@nestjs/microservices'; +import { Test, TestingModule } from '@nestjs/testing'; +import * as ProtoLoader from '@grpc/proto-loader'; +import * as GRPC from '@grpc/grpc-js'; + +import { + FileProtoFile, + FileProto, + ResetProtoFile, + FileBucketProtoFile, + FileProviderProtoFile, + FileProviderProto, +} from 'juno-proto'; +import { AppModule } from './../src/app.module'; + +const { JUNO_FILE_SERVICE_FILE_PACKAGE_NAME } = FileProto; + +let app: INestMicroservice; + +const TEST_SERVICE_ADDR = 'file-service:50003'; + +jest.setTimeout(15000); + +async function initApp() { + const moduleFixture: TestingModule = await Test.createTestingModule({ + imports: [AppModule], + }).compile(); + + const app = moduleFixture.createNestMicroservice({ + transport: Transport.GRPC, + options: { + package: [JUNO_FILE_SERVICE_FILE_PACKAGE_NAME], + protoPath: [FileProtoFile], + url: TEST_SERVICE_ADDR, + }, + }); + + await app.init(); + await app.listen(); + + return app; +} + +let fileClient: any; + +const bucketName = 'test-delete-bog-juno'; +const configId = 0; +const configEnv = 'prod'; +const providerName = 'backblazeb2-delete'; +const providerNameAzure = 'azure-delete'; + +const accessKeyId = process.env.accessKeyId; +const secretAccessKey = process.env.secretAccessKey; +const baseURL = process.env.baseURL; +const accountName = process.env.azureStorageAccountName; +const accountKey = process.env.azureStorageAccountKey; + +beforeAll(async () => { + app = await initApp(); + + const proto = ProtoLoader.loadSync([ResetProtoFile]) as any; + const protoGRPC = GRPC.loadPackageDefinition(proto) as any; + const resetClient = new protoGRPC.juno.reset_db.DatabaseReset( + process.env.DB_SERVICE_ADDR, + GRPC.credentials.createInsecure(), + ); + + await new Promise((resolve) => { + resetClient.resetDb({}, () => { + resolve(0); + }); + }); + + const fileProto = ProtoLoader.loadSync([FileProtoFile]) as any; + const fileProtoGRPC = GRPC.loadPackageDefinition(fileProto) as any; + fileClient = new fileProtoGRPC.juno.file_service.file.FileService( + TEST_SERVICE_ADDR, + GRPC.credentials.createInsecure(), + ); + + const providerProto = ProtoLoader.loadSync([FileProviderProtoFile]) as any; + const providerProtoGRPC = GRPC.loadPackageDefinition(providerProto) as any; + const providerClient = + new providerProtoGRPC.juno.file_service.provider.FileProviderDbService( + process.env.DB_SERVICE_ADDR, + GRPC.credentials.createInsecure(), + ); + + await new Promise((resolve) => { + providerClient.createProvider( + { + providerName: providerName, + accessKey: JSON.stringify({ accessKeyId, secretAccessKey }), + metadata: JSON.stringify({ endpoint: baseURL }), + bucket: [], + type: FileProviderProto.ProviderType.S3, + }, + () => resolve(0), + ); + }); + + await new Promise((resolve) => { + providerClient.createProvider( + { + providerName: providerNameAzure, + accessKey: JSON.stringify({ accountName, accountKey }), + metadata: JSON.stringify({ endpoint: '' }), + bucket: [], + type: FileProviderProto.ProviderType.AZURE, + }, + () => resolve(0), + ); + }); + + const bucketProto = ProtoLoader.loadSync([FileBucketProtoFile]) as any; + const bucketProtoGRPC = GRPC.loadPackageDefinition(bucketProto) as any; + const bucketClient = + new bucketProtoGRPC.juno.file_service.bucket.BucketDbService( + process.env.DB_SERVICE_ADDR, + GRPC.credentials.createInsecure(), + ); + + await new Promise((resolve) => { + bucketClient.createBucket( + { + name: bucketName, + configId, + configEnv, + fileProviderName: providerName, + files: [], + }, + () => resolve(0), + ); + }); + + await new Promise((resolve) => { + bucketClient.createBucket( + { + name: bucketName + '-azure', + configId, + configEnv, + fileProviderName: providerNameAzure, + files: [], + }, + () => resolve(0), + ); + }); +}); + +afterAll(async () => { + await app.close(); +}); + +describe('File Service Delete Files Tests', () => { + it('Fails to delete with missing bucketName', async () => { + const promise = new Promise((resolve) => { + fileClient.deleteFiles( + { + bucketName: '', + configId, + configEnv, + fileNames: ['SomeFile'], + }, + (err: any) => { + expect(err).not.toBeNull(); + resolve({}); + }, + ); + }); + + await promise; + }); + + it('Fails to delete with empty fileNames', async () => { + const promise = new Promise((resolve) => { + fileClient.deleteFiles( + { + bucketName, + configId, + configEnv, + fileNames: [], + }, + (err: any) => { + expect(err).not.toBeNull(); + resolve({}); + }, + ); + }); + + await promise; + }); + + it('Deletes files correctly - S3', async () => { + // Upload a file first + await new Promise((resolve) => { + fileClient.uploadFile( + { + bucketName, + fileName: 'FileToDelete', + providerName, + configId, + configEnv, + region: 'us-east-005', + }, + (err: any) => { + expect(err).toBeNull(); + resolve({}); + }, + ); + }); + + // Now delete it + const promise = new Promise((resolve) => { + fileClient.deleteFiles( + { + bucketName, + configId, + configEnv, + fileNames: ['FileToDelete'], + }, + (err: any, response: any) => { + expect(err).toBeNull(); + expect(response.bucketName).toBe(bucketName); + expect(response.fileNames).toContain('FileToDelete'); + resolve({}); + }, + ); + }); + + await promise; + }); + + it('Deletes files correctly - Azure', async () => { + // Upload a file first + await new Promise((resolve) => { + fileClient.uploadFile( + { + bucketName: bucketName + '-azure', + fileName: 'FileToDeleteAzure', + providerName: providerNameAzure, + configId, + configEnv, + region: 'us-east-005', + }, + (err: any) => { + expect(err).toBeNull(); + resolve({}); + }, + ); + }); + + // Now delete it + const promise = new Promise((resolve) => { + fileClient.deleteFiles( + { + bucketName: bucketName + '-azure', + configId, + configEnv, + fileNames: ['FileToDeleteAzure'], + }, + (err: any, response: any) => { + expect(err).toBeNull(); + expect(response.bucketName).toBe(bucketName + '-azure'); + expect(response.fileNames).toContain('FileToDeleteAzure'); + resolve({}); + }, + ); + }); + + await promise; + }); +}); From 767fb5adaab71c81b7c3764631c9a17c96a4e4b6 Mon Sep 17 00:00:00 2001 From: asarode9 Date: Tue, 7 Apr 2026 05:28:26 -0400 Subject: [PATCH 3/8] Prettier fix --- .../src/modules/file_transfer/s3_handler.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/file-service/src/modules/file_transfer/s3_handler.ts b/packages/file-service/src/modules/file_transfer/s3_handler.ts index 73d9b7ab..a5c5e7cc 100644 --- a/packages/file-service/src/modules/file_transfer/s3_handler.ts +++ b/packages/file-service/src/modules/file_transfer/s3_handler.ts @@ -93,10 +93,16 @@ export class S3FileHandler { const versions = await s3Client.send(listCommand); for (const version of versions.Versions ?? []) { - objectsToDelete.push({ Key: version.Key!, VersionId: version.VersionId }); + objectsToDelete.push({ + Key: version.Key!, + VersionId: version.VersionId, + }); } for (const marker of versions.DeleteMarkers ?? []) { - objectsToDelete.push({ Key: marker.Key!, VersionId: marker.VersionId }); + objectsToDelete.push({ + Key: marker.Key!, + VersionId: marker.VersionId, + }); } } From 60e05eab38b785f8122adfad07f6e680826afa79 Mon Sep 17 00:00:00 2001 From: asarode9 Date: Tue, 7 Apr 2026 05:39:03 -0400 Subject: [PATCH 4/8] made greptile fixes --- packages/api-gateway/src/models/file_delete.dto.ts | 10 +++++++++- .../src/modules/file_transfer/s3_handler.ts | 9 ++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/api-gateway/src/models/file_delete.dto.ts b/packages/api-gateway/src/models/file_delete.dto.ts index 3f0a10d3..9540a67b 100644 --- a/packages/api-gateway/src/models/file_delete.dto.ts +++ b/packages/api-gateway/src/models/file_delete.dto.ts @@ -1,5 +1,11 @@ import { ApiProperty } from '@nestjs/swagger'; -import { IsArray, IsInt, IsNotEmpty, IsString, Min } from 'class-validator'; +import { + IsArray, + IsInt, + IsNotEmpty, + IsString, + Min, +} from 'class-validator'; import { FileProto } from 'juno-proto'; export class DeleteFilesModel { @@ -26,6 +32,8 @@ export class DeleteFilesModel { }) @IsArray() @IsNotEmpty() + @IsString({ each: true }) + @IsNotEmpty({ each: true }) fileNames: string[]; } diff --git a/packages/file-service/src/modules/file_transfer/s3_handler.ts b/packages/file-service/src/modules/file_transfer/s3_handler.ts index a5c5e7cc..89fb7e7a 100644 --- a/packages/file-service/src/modules/file_transfer/s3_handler.ts +++ b/packages/file-service/src/modules/file_transfer/s3_handler.ts @@ -111,7 +111,14 @@ export class S3FileHandler { Bucket: bucket, Delete: { Objects: objectsToDelete }, }); - await s3Client.send(deleteCommand); + const result = await s3Client.send(deleteCommand); + if (result.Errors && result.Errors.length > 0) { + const failedKeys = result.Errors.map((e) => e.Key).join(', '); + throw new RpcException({ + code: status.FAILED_PRECONDITION, + message: `Failed to delete some objects from S3: ${failedKeys}`, + }); + } } } catch (err) { throw new RpcException({ From d520126c33e6b760ab7fad5e26630064793ee440 Mon Sep 17 00:00:00 2001 From: asarode9 Date: Tue, 7 Apr 2026 05:41:07 -0400 Subject: [PATCH 5/8] prettier fix --- packages/api-gateway/src/models/file_delete.dto.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/packages/api-gateway/src/models/file_delete.dto.ts b/packages/api-gateway/src/models/file_delete.dto.ts index 9540a67b..dad55901 100644 --- a/packages/api-gateway/src/models/file_delete.dto.ts +++ b/packages/api-gateway/src/models/file_delete.dto.ts @@ -1,11 +1,5 @@ import { ApiProperty } from '@nestjs/swagger'; -import { - IsArray, - IsInt, - IsNotEmpty, - IsString, - Min, -} from 'class-validator'; +import { IsArray, IsInt, IsNotEmpty, IsString, Min } from 'class-validator'; import { FileProto } from 'juno-proto'; export class DeleteFilesModel { From d6d2742ee08aa6e1f58ce576a2ab4a0fe8524ae4 Mon Sep 17 00:00:00 2001 From: asarode9 Date: Tue, 7 Apr 2026 08:11:31 -0400 Subject: [PATCH 6/8] Fixed e2e test errors --- .../api-gateway/test/file_delete.e2e-spec.ts | 42 +++++++++++++--- .../modules/file_transfer/azure_handler.ts | 2 + .../src/modules/file_transfer/s3_handler.ts | 50 ++++--------------- 3 files changed, 47 insertions(+), 47 deletions(-) diff --git a/packages/api-gateway/test/file_delete.e2e-spec.ts b/packages/api-gateway/test/file_delete.e2e-spec.ts index 98916a9b..aa387601 100644 --- a/packages/api-gateway/test/file_delete.e2e-spec.ts +++ b/packages/api-gateway/test/file_delete.e2e-spec.ts @@ -1,3 +1,4 @@ +import { CreateBucketCommand, DeleteBucketCommand, S3Client } from '@aws-sdk/client-s3'; import { Test, TestingModule } from '@nestjs/testing'; import { ClassSerializerInterceptor, @@ -18,6 +19,8 @@ import * as GRPC from '@grpc/grpc-js'; import * as ProtoLoader from '@grpc/proto-loader'; import { RpcExceptionFilter } from 'src/rpc_exception_filter'; +jest.setTimeout(15000); + let app: INestApplication; const ADMIN_EMAIL = 'test-superadmin@test.com'; const ADMIN_PASSWORD = 'test-password'; @@ -38,7 +41,7 @@ async function APIKeyForProjectName(projectName: string): Promise { return key.body['apiKey']; } -const bucketName = 'test-delete-gw-juno'; +const bucketName = 'test-delete-bog-juno'; const configId = 0; const providerName = 'backblazeb2-delete-gw'; @@ -46,6 +49,9 @@ const accessKeyId = process.env.accessKeyId; const secretAccessKey = process.env.secretAccessKey; const baseURL = process.env.baseURL; +const s3BucketName = `${bucketName}-${configId}-prod`; +let s3Client: S3Client; + beforeAll(async () => { const proto = ProtoLoader.loadSync([ResetProtoFile]) as any; const protoGRPC = GRPC.loadPackageDefinition(proto) as any; @@ -98,11 +104,32 @@ beforeAll(async () => { () => resolve(0), ); }); + + s3Client = new S3Client({ + endpoint: baseURL as string, + region: 'us-east-005', + credentials: { + accessKeyId: accessKeyId as string, + secretAccessKey: secretAccessKey as string, + }, + forcePathStyle: true, + }); + try { + await s3Client.send(new CreateBucketCommand({ Bucket: s3BucketName })); + } catch { + // Bucket may already exist from a previous run + } }); -afterAll((done) => { - app.close(); - done(); +afterAll(async () => { + if (app) await app.close(); + if (s3Client) { + try { + await s3Client.send(new DeleteBucketCommand({ Bucket: s3BucketName })); + } catch { + // Ignore cleanup errors + } + } }); beforeEach(async () => { @@ -198,14 +225,15 @@ describe('File Delete Routes', () => { ); }); - return request(app.getHttpServer()) + const res = await request(app.getHttpServer()) .delete('/file/delete') .set('Authorization', 'Bearer ' + apiKey) .send({ bucketName, configId, fileNames: ['FileToDeleteGW'], - }) - .expect(200); + }); + + expect(res.status).toBe(200); }); }); diff --git a/packages/file-service/src/modules/file_transfer/azure_handler.ts b/packages/file-service/src/modules/file_transfer/azure_handler.ts index 332e7073..e93948b7 100644 --- a/packages/file-service/src/modules/file_transfer/azure_handler.ts +++ b/packages/file-service/src/modules/file_transfer/azure_handler.ts @@ -55,6 +55,8 @@ export class AzureFileHandler { const containerClient = blobServiceClient.getContainerClient( `${request.bucketName}-${request.configId}-${request.configEnv}`, ); + const containerExists = await containerClient.exists(); + if (!containerExists) return; await Promise.all( request.fileNames.map((name) => containerClient.getBlobClient(name).deleteIfExists(), diff --git a/packages/file-service/src/modules/file_transfer/s3_handler.ts b/packages/file-service/src/modules/file_transfer/s3_handler.ts index 89fb7e7a..b36a1a57 100644 --- a/packages/file-service/src/modules/file_transfer/s3_handler.ts +++ b/packages/file-service/src/modules/file_transfer/s3_handler.ts @@ -3,7 +3,6 @@ import { GetObjectCommand, PutObjectCommand, DeleteObjectsCommand, - ListObjectVersionsCommand, } from '@aws-sdk/client-s3'; import { FileProto, FileProviderProto } from 'juno-proto'; import { RpcException } from '@nestjs/microservices'; @@ -81,46 +80,17 @@ export class S3FileHandler { async deleteFiles(request: FileProto.DeleteFilesRequest): Promise { try { const s3Client = await this.getS3Client('us-east-005'); - const bucket = `${request.bucketName}-${request.configId}-${request.configEnv}`; - - const objectsToDelete: { Key: string; VersionId?: string }[] = []; - - for (const name of request.fileNames) { - const listCommand = new ListObjectVersionsCommand({ - Bucket: bucket, - Prefix: name, - }); - const versions = await s3Client.send(listCommand); - - for (const version of versions.Versions ?? []) { - objectsToDelete.push({ - Key: version.Key!, - VersionId: version.VersionId, - }); - } - for (const marker of versions.DeleteMarkers ?? []) { - objectsToDelete.push({ - Key: marker.Key!, - VersionId: marker.VersionId, - }); - } - } - - if (objectsToDelete.length > 0) { - const deleteCommand = new DeleteObjectsCommand({ - Bucket: bucket, - Delete: { Objects: objectsToDelete }, - }); - const result = await s3Client.send(deleteCommand); - if (result.Errors && result.Errors.length > 0) { - const failedKeys = result.Errors.map((e) => e.Key).join(', '); - throw new RpcException({ - code: status.FAILED_PRECONDITION, - message: `Failed to delete some objects from S3: ${failedKeys}`, - }); - } - } + const deleteCommand = new DeleteObjectsCommand({ + Bucket: `${request.bucketName}-${request.configId}-${request.configEnv}`, + Delete: { + Objects: request.fileNames.map((name) => ({ Key: name })), + }, + }); + await s3Client.send(deleteCommand); } catch (err) { + if ((err as any).name === 'NoSuchBucket') { + return; + } throw new RpcException({ code: status.FAILED_PRECONDITION, message: `Failed to delete files from S3: ${err}`, From 72acc44595583401e32ddd1c66a3ef71fc7615b1 Mon Sep 17 00:00:00 2001 From: asarode9 Date: Tue, 7 Apr 2026 08:12:13 -0400 Subject: [PATCH 7/8] prettier fix --- packages/api-gateway/test/file_delete.e2e-spec.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/api-gateway/test/file_delete.e2e-spec.ts b/packages/api-gateway/test/file_delete.e2e-spec.ts index aa387601..e8fb3525 100644 --- a/packages/api-gateway/test/file_delete.e2e-spec.ts +++ b/packages/api-gateway/test/file_delete.e2e-spec.ts @@ -1,4 +1,8 @@ -import { CreateBucketCommand, DeleteBucketCommand, S3Client } from '@aws-sdk/client-s3'; +import { + CreateBucketCommand, + DeleteBucketCommand, + S3Client, +} from '@aws-sdk/client-s3'; import { Test, TestingModule } from '@nestjs/testing'; import { ClassSerializerInterceptor, From 333f243dd0ecee500132cf00989e29d40f721db5 Mon Sep 17 00:00:00 2001 From: asarode9 Date: Tue, 7 Apr 2026 08:32:53 -0400 Subject: [PATCH 8/8] Hopefully fixed api gateway issue --- .../api-gateway/test/file_delete.e2e-spec.ts | 5 +- .../file-service/test/file_delete.e2e-spec.ts | 273 ------------------ 2 files changed, 3 insertions(+), 275 deletions(-) delete mode 100644 packages/file-service/test/file_delete.e2e-spec.ts diff --git a/packages/api-gateway/test/file_delete.e2e-spec.ts b/packages/api-gateway/test/file_delete.e2e-spec.ts index e8fb3525..aa706f10 100644 --- a/packages/api-gateway/test/file_delete.e2e-spec.ts +++ b/packages/api-gateway/test/file_delete.e2e-spec.ts @@ -45,9 +45,10 @@ async function APIKeyForProjectName(projectName: string): Promise { return key.body['apiKey']; } -const bucketName = 'test-delete-bog-juno'; +const runSuffix = Math.random().toString(36).slice(2, 8); +const bucketName = `test-delete-bog-juno-${runSuffix}`; const configId = 0; -const providerName = 'backblazeb2-delete-gw'; +const providerName = `backblazeb2-delete-gw-${runSuffix}`; const accessKeyId = process.env.accessKeyId; const secretAccessKey = process.env.secretAccessKey; diff --git a/packages/file-service/test/file_delete.e2e-spec.ts b/packages/file-service/test/file_delete.e2e-spec.ts deleted file mode 100644 index 643d6788..00000000 --- a/packages/file-service/test/file_delete.e2e-spec.ts +++ /dev/null @@ -1,273 +0,0 @@ -import { INestMicroservice } from '@nestjs/common'; -import { MicroserviceOptions, Transport } from '@nestjs/microservices'; -import { Test, TestingModule } from '@nestjs/testing'; -import * as ProtoLoader from '@grpc/proto-loader'; -import * as GRPC from '@grpc/grpc-js'; - -import { - FileProtoFile, - FileProto, - ResetProtoFile, - FileBucketProtoFile, - FileProviderProtoFile, - FileProviderProto, -} from 'juno-proto'; -import { AppModule } from './../src/app.module'; - -const { JUNO_FILE_SERVICE_FILE_PACKAGE_NAME } = FileProto; - -let app: INestMicroservice; - -const TEST_SERVICE_ADDR = 'file-service:50003'; - -jest.setTimeout(15000); - -async function initApp() { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - const app = moduleFixture.createNestMicroservice({ - transport: Transport.GRPC, - options: { - package: [JUNO_FILE_SERVICE_FILE_PACKAGE_NAME], - protoPath: [FileProtoFile], - url: TEST_SERVICE_ADDR, - }, - }); - - await app.init(); - await app.listen(); - - return app; -} - -let fileClient: any; - -const bucketName = 'test-delete-bog-juno'; -const configId = 0; -const configEnv = 'prod'; -const providerName = 'backblazeb2-delete'; -const providerNameAzure = 'azure-delete'; - -const accessKeyId = process.env.accessKeyId; -const secretAccessKey = process.env.secretAccessKey; -const baseURL = process.env.baseURL; -const accountName = process.env.azureStorageAccountName; -const accountKey = process.env.azureStorageAccountKey; - -beforeAll(async () => { - app = await initApp(); - - const proto = ProtoLoader.loadSync([ResetProtoFile]) as any; - const protoGRPC = GRPC.loadPackageDefinition(proto) as any; - const resetClient = new protoGRPC.juno.reset_db.DatabaseReset( - process.env.DB_SERVICE_ADDR, - GRPC.credentials.createInsecure(), - ); - - await new Promise((resolve) => { - resetClient.resetDb({}, () => { - resolve(0); - }); - }); - - const fileProto = ProtoLoader.loadSync([FileProtoFile]) as any; - const fileProtoGRPC = GRPC.loadPackageDefinition(fileProto) as any; - fileClient = new fileProtoGRPC.juno.file_service.file.FileService( - TEST_SERVICE_ADDR, - GRPC.credentials.createInsecure(), - ); - - const providerProto = ProtoLoader.loadSync([FileProviderProtoFile]) as any; - const providerProtoGRPC = GRPC.loadPackageDefinition(providerProto) as any; - const providerClient = - new providerProtoGRPC.juno.file_service.provider.FileProviderDbService( - process.env.DB_SERVICE_ADDR, - GRPC.credentials.createInsecure(), - ); - - await new Promise((resolve) => { - providerClient.createProvider( - { - providerName: providerName, - accessKey: JSON.stringify({ accessKeyId, secretAccessKey }), - metadata: JSON.stringify({ endpoint: baseURL }), - bucket: [], - type: FileProviderProto.ProviderType.S3, - }, - () => resolve(0), - ); - }); - - await new Promise((resolve) => { - providerClient.createProvider( - { - providerName: providerNameAzure, - accessKey: JSON.stringify({ accountName, accountKey }), - metadata: JSON.stringify({ endpoint: '' }), - bucket: [], - type: FileProviderProto.ProviderType.AZURE, - }, - () => resolve(0), - ); - }); - - const bucketProto = ProtoLoader.loadSync([FileBucketProtoFile]) as any; - const bucketProtoGRPC = GRPC.loadPackageDefinition(bucketProto) as any; - const bucketClient = - new bucketProtoGRPC.juno.file_service.bucket.BucketDbService( - process.env.DB_SERVICE_ADDR, - GRPC.credentials.createInsecure(), - ); - - await new Promise((resolve) => { - bucketClient.createBucket( - { - name: bucketName, - configId, - configEnv, - fileProviderName: providerName, - files: [], - }, - () => resolve(0), - ); - }); - - await new Promise((resolve) => { - bucketClient.createBucket( - { - name: bucketName + '-azure', - configId, - configEnv, - fileProviderName: providerNameAzure, - files: [], - }, - () => resolve(0), - ); - }); -}); - -afterAll(async () => { - await app.close(); -}); - -describe('File Service Delete Files Tests', () => { - it('Fails to delete with missing bucketName', async () => { - const promise = new Promise((resolve) => { - fileClient.deleteFiles( - { - bucketName: '', - configId, - configEnv, - fileNames: ['SomeFile'], - }, - (err: any) => { - expect(err).not.toBeNull(); - resolve({}); - }, - ); - }); - - await promise; - }); - - it('Fails to delete with empty fileNames', async () => { - const promise = new Promise((resolve) => { - fileClient.deleteFiles( - { - bucketName, - configId, - configEnv, - fileNames: [], - }, - (err: any) => { - expect(err).not.toBeNull(); - resolve({}); - }, - ); - }); - - await promise; - }); - - it('Deletes files correctly - S3', async () => { - // Upload a file first - await new Promise((resolve) => { - fileClient.uploadFile( - { - bucketName, - fileName: 'FileToDelete', - providerName, - configId, - configEnv, - region: 'us-east-005', - }, - (err: any) => { - expect(err).toBeNull(); - resolve({}); - }, - ); - }); - - // Now delete it - const promise = new Promise((resolve) => { - fileClient.deleteFiles( - { - bucketName, - configId, - configEnv, - fileNames: ['FileToDelete'], - }, - (err: any, response: any) => { - expect(err).toBeNull(); - expect(response.bucketName).toBe(bucketName); - expect(response.fileNames).toContain('FileToDelete'); - resolve({}); - }, - ); - }); - - await promise; - }); - - it('Deletes files correctly - Azure', async () => { - // Upload a file first - await new Promise((resolve) => { - fileClient.uploadFile( - { - bucketName: bucketName + '-azure', - fileName: 'FileToDeleteAzure', - providerName: providerNameAzure, - configId, - configEnv, - region: 'us-east-005', - }, - (err: any) => { - expect(err).toBeNull(); - resolve({}); - }, - ); - }); - - // Now delete it - const promise = new Promise((resolve) => { - fileClient.deleteFiles( - { - bucketName: bucketName + '-azure', - configId, - configEnv, - fileNames: ['FileToDeleteAzure'], - }, - (err: any, response: any) => { - expect(err).toBeNull(); - expect(response.bucketName).toBe(bucketName + '-azure'); - expect(response.fileNames).toContain('FileToDeleteAzure'); - resolve({}); - }, - ); - }); - - await promise; - }); -});