diff --git a/packages/api-gateway/src/models/file_bucket.dto.ts b/packages/api-gateway/src/models/file_bucket.dto.ts index efd27d8c..9bd93630 100644 --- a/packages/api-gateway/src/models/file_bucket.dto.ts +++ b/packages/api-gateway/src/models/file_bucket.dto.ts @@ -109,3 +109,11 @@ export class FileBucket implements FileBucketProto.Bucket { this.FileServiceFile = fileBucket.FileServiceFile ?? []; } } + +export class BucketWithFiles { + [bucketName: string]: string[]; + + constructor(entry: FileBucketProto.Files) { + this[entry.bucketName] = entry.files; + } +} diff --git a/packages/api-gateway/src/modules/file_bucket/file_bucket.controller.ts b/packages/api-gateway/src/modules/file_bucket/file_bucket.controller.ts index 74471455..454a76c0 100644 --- a/packages/api-gateway/src/modules/file_bucket/file_bucket.controller.ts +++ b/packages/api-gateway/src/modules/file_bucket/file_bucket.controller.ts @@ -23,6 +23,7 @@ import { AuthCommonProto, FileBucketProto } from 'juno-proto'; import { lastValueFrom } from 'rxjs'; import { ApiKey } from 'src/decorators/api_key.decorator'; import { + BucketWithFiles, DeleteFileBucketModel, FileBucket, RegisterFileBucketModel, @@ -132,4 +133,33 @@ export class FileBucketController implements OnModuleInit { return bucketsData.buckets?.map((bucket) => new FileBucket(bucket)) ?? []; } + + @Get('all/:configId') + @ApiOperation({ summary: 'Get all files from all buckets for a config.' }) + @ApiBadRequestResponse({ description: 'Parameters are invalid' }) + @ApiUnauthorizedResponse({ description: 'Unauthorized' }) + @ApiOkResponse({ + description: 'Returned all files grouped by bucket name', + type: [BucketWithFiles], + }) + async getAllFiles( + @ApiKey() apiKey: AuthCommonProto.ApiKey, + @Param('configId') configId: string, + ): Promise { + const id = parseInt(configId); + if (Number.isNaN(id) || id < 0) { + throw new BadRequestException( + 'Id must be an int greater than or equal to 0', + ); + } + + const grpcResponse = this.fileBucketService.getAllFiles({ + configId: id, + configEnv: apiKey.environment, + }); + + const data = await lastValueFrom(grpcResponse); + + return (data.files ?? []).map((entry) => new BucketWithFiles(entry)); + } } diff --git a/packages/api-gateway/src/modules/file_bucket/file_bucket.module.ts b/packages/api-gateway/src/modules/file_bucket/file_bucket.module.ts index 050cc462..656e531a 100644 --- a/packages/api-gateway/src/modules/file_bucket/file_bucket.module.ts +++ b/packages/api-gateway/src/modules/file_bucket/file_bucket.module.ts @@ -72,6 +72,8 @@ import { }) export class FileBucketModule implements NestModule { configure(consumer: MiddlewareConsumer) { - consumer.apply(ApiKeyMiddleware).forRoutes('/file/bucket', 'file/bucket/*'); + consumer + .apply(ApiKeyMiddleware) + .forRoutes('/file/bucket', 'file/bucket/*', '/file/all/*'); } } diff --git a/packages/api-gateway/test/file_bucket.e2e-spec.ts b/packages/api-gateway/test/file_bucket.e2e-spec.ts index 5ff19c78..1d164851 100644 --- a/packages/api-gateway/test/file_bucket.e2e-spec.ts +++ b/packages/api-gateway/test/file_bucket.e2e-spec.ts @@ -230,4 +230,22 @@ describe('File Bucket Routes', () => { await client.send(command); }); + + it('Gets all files successfully', async () => { + await request(app.getHttpServer()) + .get('/file/all/0') + .set('Authorization', 'Bearer ' + apiKey) + .expect(200); + }); + + it('Unsuccessfully gets all files due to invalid configId', async () => { + await request(app.getHttpServer()) + .get('/file/all/invalid') + .set('Authorization', 'Bearer ' + apiKey) + .expect(400); + }); + + it('Unsuccessfully gets all files due to missing authorization', async () => { + await request(app.getHttpServer()).get('/file/all/0').expect(401); + }); }); diff --git a/packages/file-service/src/modules/file_bucket/azure_handler.ts b/packages/file-service/src/modules/file_bucket/azure_handler.ts index 8d9dd5f3..b9aad4e6 100644 --- a/packages/file-service/src/modules/file_bucket/azure_handler.ts +++ b/packages/file-service/src/modules/file_bucket/azure_handler.ts @@ -68,4 +68,25 @@ export class AzureBucketHandler { }); } } + + async listFiles( + request: FileBucketProto.GetBucketRequest, + ): Promise { + try { + const blobClient = await this.getBlobServiceClient(); + const containerClient = blobClient.getContainerClient( + `${request.name}-${request.configId}-${request.configEnv}`, + ); + const files: string[] = []; + for await (const blob of containerClient.listBlobsFlat()) { + files.push(blob.name); + } + return files; + } catch (error) { + throw new RpcException({ + code: status.FAILED_PRECONDITION, + message: `Failed to list files: ${error.message} `, + }); + } + } } diff --git a/packages/file-service/src/modules/file_bucket/file_bucket.controller.ts b/packages/file-service/src/modules/file_bucket/file_bucket.controller.ts index 52b47d73..fb301c04 100644 --- a/packages/file-service/src/modules/file_bucket/file_bucket.controller.ts +++ b/packages/file-service/src/modules/file_bucket/file_bucket.controller.ts @@ -20,4 +20,10 @@ export class FileBucketController ): Promise { return await this.fileBucketService.removeBucket(request); } + + async getAllFiles( + request: FileBucketProto.GetAllFilesRequest, + ): Promise { + return await this.fileBucketService.getAllFiles(request); + } } diff --git a/packages/file-service/src/modules/file_bucket/file_bucket.service.ts b/packages/file-service/src/modules/file_bucket/file_bucket.service.ts index 0b675da5..416454db 100644 --- a/packages/file-service/src/modules/file_bucket/file_bucket.service.ts +++ b/packages/file-service/src/modules/file_bucket/file_bucket.service.ts @@ -1,5 +1,5 @@ import { status } from '@grpc/grpc-js'; -import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; +import { Inject, Injectable, Logger, OnModuleInit } from '@nestjs/common'; import { ClientGrpc, RpcException } from '@nestjs/microservices'; import { FileBucketProto, FileProviderProto } from 'juno-proto'; import { lastValueFrom } from 'rxjs'; @@ -8,6 +8,7 @@ import { S3BucketHandler } from './s3_handler'; @Injectable() export class FileBucketService implements OnModuleInit { + private readonly logger = new Logger(FileBucketService.name); private fileBucketDBService: FileBucketProto.BucketDbServiceClient; private fileProviderDBService: FileProviderProto.FileProviderDbServiceClient; @@ -134,4 +135,73 @@ export class FileBucketService implements OnModuleInit { }); } } + + async getAllFiles( + request: FileBucketProto.GetAllFilesRequest, + ): Promise { + try { + const buckets = await lastValueFrom( + this.fileBucketDBService.getBucketsByConfigIdAndEnv({ + configId: request.configId, + configEnv: request.configEnv, + }), + ); + + const results = await Promise.all( + (buckets.buckets ?? []).map(async (bucket) => { + try { + const provider = await this.getProvider(bucket.fileProviderName); + const bucketRequest: FileBucketProto.GetBucketRequest = { + name: bucket.name, + configId: bucket.configId, + configEnv: bucket.configEnv, + }; + + let fileList: string[]; + switch (provider.providerType) { + case FileProviderProto.ProviderType.S3: { + const handler = new S3BucketHandler(provider); + fileList = await handler.listFiles(bucketRequest); + break; + } + case FileProviderProto.ProviderType.AZURE: { + const handler = new AzureBucketHandler(provider); + fileList = await handler.listFiles(bucketRequest); + break; + } + default: + throw new RpcException({ + code: status.FAILED_PRECONDITION, + message: `Unsupported provider type: ${provider.providerType} `, + }); + } + + return { + bucketName: bucket.name, + files: fileList, + }; + } catch (error) { + this.logger.error( + `Failed to list files for bucket "${bucket.name}": ${error?.message}`, + ); + return null; + } + }), + ); + + const files = results.filter( + (result): result is FileBucketProto.Files => result !== null, + ); + + return { files }; + } catch (error) { + if (error instanceof RpcException) { + throw error; + } + throw new RpcException({ + code: status.FAILED_PRECONDITION, + message: `Failed to get all files: ${error.message} `, + }); + } + } } diff --git a/packages/file-service/src/modules/file_bucket/s3_handler.ts b/packages/file-service/src/modules/file_bucket/s3_handler.ts index 91ff6888..0b12e77f 100644 --- a/packages/file-service/src/modules/file_bucket/s3_handler.ts +++ b/packages/file-service/src/modules/file_bucket/s3_handler.ts @@ -1,6 +1,7 @@ import { CreateBucketCommand, DeleteBucketCommand, + ListObjectsV2Command, S3Client, } from '@aws-sdk/client-s3'; import { status } from '@grpc/grpc-js'; @@ -67,4 +68,37 @@ export class S3BucketHandler { }); } } + + async listFiles( + request: FileBucketProto.GetBucketRequest, + ): Promise { + try { + const s3Client = await this.getS3Client('us-east-005'); + const files: string[] = []; + let continuationToken: string | undefined; + + do { + const listObjectsV2Command = new ListObjectsV2Command({ + Bucket: `${request.name}-${request.configId}-${request.configEnv}`, + ContinuationToken: continuationToken, + }); + const response = await s3Client.send(listObjectsV2Command); + if (response.Contents) { + for (const object of response.Contents) { + if (object.Key) { + files.push(object.Key); + } + } + } + continuationToken = response.NextContinuationToken; + } while (continuationToken); + + return files; + } catch (error) { + throw new RpcException({ + code: status.FAILED_PRECONDITION, + message: `Failed to list files: ${error.message} `, + }); + } + } } diff --git a/packages/file-service/test/file_bucket.e2e-spec.ts b/packages/file-service/test/file_bucket.e2e-spec.ts index 7896242e..c79bc3e7 100644 --- a/packages/file-service/test/file_bucket.e2e-spec.ts +++ b/packages/file-service/test/file_bucket.e2e-spec.ts @@ -267,3 +267,91 @@ describe('File Bucket Creation Tests', () => { await createBucketPromise2; }); }); + +describe('Get All Files Tests', () => { + it('Successfully gets all files for a config', async () => { + const metadata = { + endpoint: baseURL, + region: region, + credentials: { + accessKeyId: accessKeyId as string, + secretAccessKey: secretAccessKey as string, + }, + }; + + const client = new S3Client(metadata); + const command = new DeleteBucketCommand({ + Bucket: `get-all-files-bucket-${configId}-${configEnv}`, + }); + + try { + await client.send(command); + } catch {} + + await new Promise((resolve, reject) => { + bucketClient.registerBucket( + { + name: 'get-all-files-bucket', + configId, + configEnv, + fileProviderName: providerName, + }, + (err: any) => { + if (err) return reject(err); + resolve(0); + }, + ); + }); + + const getAllFilesPromise = new Promise((resolve, reject) => { + bucketClient.getAllFiles( + { + configId, + configEnv, + }, + (err: any, response: any) => { + if (err) return reject(err); + expect(err).toBeNull(); + expect(response).toBeDefined(); + expect(response.files).toBeDefined(); + resolve(response); + }, + ); + }); + await getAllFilesPromise; + + await new Promise((resolve, reject) => { + bucketClient.removeBucket( + { + name: 'get-all-files-bucket', + configId, + configEnv, + fileProviderName: providerName, + }, + (err: any) => { + if (err) return reject(err); + resolve(0); + }, + ); + }); + }); + + it('Returns empty files for nonexistent config', async () => { + const getAllFilesPromise = new Promise((resolve, reject) => { + bucketClient.getAllFiles( + { + configId: 9999, + configEnv: 'nonexistent', + }, + (err: any, response: any) => { + if (err) return reject(err); + expect(err).toBeNull(); + expect(response).toBeDefined(); + expect(response.files ?? []).toEqual([]); + resolve(response); + }, + ); + }); + await getAllFilesPromise; + }); +}); diff --git a/packages/proto/definitions/file_bucket.proto b/packages/proto/definitions/file_bucket.proto index 82b22259..07adec2f 100644 --- a/packages/proto/definitions/file_bucket.proto +++ b/packages/proto/definitions/file_bucket.proto @@ -16,6 +16,7 @@ service BucketDbService { service BucketFileService { rpc registerBucket(RegisterBucketRequest) returns (Bucket); rpc removeBucket(RemoveBucketRequest) returns (Bucket); + rpc getAllFiles(GetAllFilesRequest) returns (GetAllFilesResponse); } message Bucket { @@ -76,3 +77,17 @@ message RemoveBucketRequest { string configEnv = 3; string fileProviderName = 4; } + +message GetAllFilesRequest { + int32 configId = 1; + string configEnv = 2; +} + +message Files { + string bucketName = 1; + repeated string files = 2; +} + +message GetAllFilesResponse { + repeated Files files = 1; +}