Andrey - Get All Files

packages/api-gateway/src/models/file_bucket.dto.ts

@@ -109,3 +109,11 @@ export class FileBucket implements FileBucketProto.Bucket {
     this.FileServiceFile = fileBucket.FileServiceFile ?? [];
   }
 }
+
+export class BucketWithFiles {
+  [bucketName: string]: string[];
+
+  constructor(entry: FileBucketProto.Files) {
+    this[entry.bucketName] = entry.files;
+  }
+}

packages/api-gateway/src/modules/file_bucket/file_bucket.controller.ts

@@ -23,6 +23,7 @@ import { AuthCommonProto, FileBucketProto } from 'juno-proto';
 import { lastValueFrom } from 'rxjs';
 import { ApiKey } from 'src/decorators/api_key.decorator';
 import {
+  BucketWithFiles,
   DeleteFileBucketModel,
   FileBucket,
   RegisterFileBucketModel,
@@ -132,4 +133,33 @@ export class FileBucketController implements OnModuleInit {
 
     return bucketsData.buckets?.map((bucket) => new FileBucket(bucket)) ?? [];
   }
+
+  @Get('all/:configId')
+  @ApiOperation({ summary: 'Get all files from all buckets for a config.' })
+  @ApiBadRequestResponse({ description: 'Parameters are invalid' })
+  @ApiUnauthorizedResponse({ description: 'Unauthorized' })
+  @ApiOkResponse({
+    description: 'Returned all files grouped by bucket name',
+    type: [BucketWithFiles],
+  })
+  async getAllFiles(
+    @ApiKey() apiKey: AuthCommonProto.ApiKey,
+    @Param('configId') configId: string,
+  ): Promise<BucketWithFiles[]> {
+    const id = parseInt(configId);
+    if (Number.isNaN(id) || id < 0) {
+      throw new BadRequestException(
+        'Id must be an int greater than or equal to 0',
+      );
+    }
+
+    const grpcResponse = this.fileBucketService.getAllFiles({
+      configId: id,
+      configEnv: apiKey.environment,
+    });
+
+    const data = await lastValueFrom(grpcResponse);
+
+    return (data.files ?? []).map((entry) => new BucketWithFiles(entry));
+  }
 }

packages/api-gateway/src/modules/file_bucket/file_bucket.module.ts

@@ -72,6 +72,8 @@ import {
 })
 export class FileBucketModule implements NestModule {
   configure(consumer: MiddlewareConsumer) {
-    consumer.apply(ApiKeyMiddleware).forRoutes('/file/bucket', 'file/bucket/*');
+    consumer
+      .apply(ApiKeyMiddleware)
+      .forRoutes('/file/bucket', 'file/bucket/*', '/file/all/*');
   }
 }

packages/api-gateway/test/file_bucket.e2e-spec.ts

@@ -230,4 +230,22 @@ describe('File Bucket Routes', () => {
 
     await client.send(command);
   });
+
+  it('Gets all files successfully', async () => {
+    await request(app.getHttpServer())
+      .get('/file/all/0')
+      .set('Authorization', 'Bearer ' + apiKey)
+      .expect(200);
+  });
+
+  it('Unsuccessfully gets all files due to invalid configId', async () => {
+    await request(app.getHttpServer())
+      .get('/file/all/invalid')
+      .set('Authorization', 'Bearer ' + apiKey)
+      .expect(400);
+  });
+
+  it('Unsuccessfully gets all files due to missing authorization', async () => {
+    await request(app.getHttpServer()).get('/file/all/0').expect(401);
+  });
 });

packages/file-service/src/modules/file_bucket/azure_handler.ts

@@ -68,4 +68,25 @@ export class AzureBucketHandler {
       });
     }
   }
+
+  async listFiles(
+    request: FileBucketProto.GetBucketRequest,
+  ): Promise<string[]> {
+    try {
+      const blobClient = await this.getBlobServiceClient();
+      const containerClient = blobClient.getContainerClient(
+        `${request.name}-${request.configId}-${request.configEnv}`,
+      );
+      const files: string[] = [];
+      for await (const blob of containerClient.listBlobsFlat()) {
+        files.push(blob.name);
+      }
+      return files;
+    } catch (error) {
+      throw new RpcException({
+        code: status.FAILED_PRECONDITION,
+        message: `Failed to list files: ${error.message} `,
+      });
+    }
+  }
 }

packages/file-service/src/modules/file_bucket/file_bucket.controller.ts

@@ -20,4 +20,10 @@ export class FileBucketController
   ): Promise<FileBucketProto.Bucket> {
     return await this.fileBucketService.removeBucket(request);
   }
+
+  async getAllFiles(
+    request: FileBucketProto.GetAllFilesRequest,
+  ): Promise<FileBucketProto.GetAllFilesResponse> {
+    return await this.fileBucketService.getAllFiles(request);
+  }
 }

packages/file-service/src/modules/file_bucket/file_bucket.service.ts

@@ -1,5 +1,5 @@
 import { status } from '@grpc/grpc-js';
-import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
+import { Inject, Injectable, Logger, OnModuleInit } from '@nestjs/common';
 import { ClientGrpc, RpcException } from '@nestjs/microservices';
 import { FileBucketProto, FileProviderProto } from 'juno-proto';
 import { lastValueFrom } from 'rxjs';
@@ -8,6 +8,7 @@ import { S3BucketHandler } from './s3_handler';
 
 @Injectable()
 export class FileBucketService implements OnModuleInit {
+  private readonly logger = new Logger(FileBucketService.name);
   private fileBucketDBService: FileBucketProto.BucketDbServiceClient;
   private fileProviderDBService: FileProviderProto.FileProviderDbServiceClient;
 
@@ -134,4 +135,73 @@ export class FileBucketService implements OnModuleInit {
       });
     }
   }
+
+  async getAllFiles(
+    request: FileBucketProto.GetAllFilesRequest,
+  ): Promise<FileBucketProto.GetAllFilesResponse> {
+    try {
+      const buckets = await lastValueFrom(
+        this.fileBucketDBService.getBucketsByConfigIdAndEnv({
+          configId: request.configId,
+          configEnv: request.configEnv,
+        }),
+      );
+
+      const results = await Promise.all(
+        (buckets.buckets ?? []).map(async (bucket) => {
+          try {
+            const provider = await this.getProvider(bucket.fileProviderName);
+            const bucketRequest: FileBucketProto.GetBucketRequest = {
+              name: bucket.name,
+              configId: bucket.configId,
+              configEnv: bucket.configEnv,
+            };
+
+            let fileList: string[];
+            switch (provider.providerType) {
+              case FileProviderProto.ProviderType.S3: {
+                const handler = new S3BucketHandler(provider);
+                fileList = await handler.listFiles(bucketRequest);
+                break;
+              }
+              case FileProviderProto.ProviderType.AZURE: {
+                const handler = new AzureBucketHandler(provider);
+                fileList = await handler.listFiles(bucketRequest);
+                break;
+              }
+              default:
+                throw new RpcException({
+                  code: status.FAILED_PRECONDITION,
+                  message: `Unsupported provider type: ${provider.providerType} `,
+                });
+            }
+
+            return {
+              bucketName: bucket.name,
+              files: fileList,
+            };
+          } catch (error) {
+            this.logger.error(
+              `Failed to list files for bucket "${bucket.name}": ${error?.message}`,
+            );
+            return null;
+          }
+        }),
+      );
+
+      const files = results.filter(
+        (result): result is FileBucketProto.Files => result !== null,
+      );
+
+      return { files };
+    } catch (error) {
+      if (error instanceof RpcException) {
+        throw error;
+      }
+      throw new RpcException({
+        code: status.FAILED_PRECONDITION,
+        message: `Failed to get all files: ${error.message} `,
+      });
+    }
+  }
 }

packages/file-service/src/modules/file_bucket/s3_handler.ts

@@ -1,6 +1,7 @@
 import {
   CreateBucketCommand,
   DeleteBucketCommand,
+  ListObjectsV2Command,
   S3Client,
 } from '@aws-sdk/client-s3';
 import { status } from '@grpc/grpc-js';
@@ -67,4 +68,37 @@ export class S3BucketHandler {
       });
     }
   }
+
+  async listFiles(
+    request: FileBucketProto.GetBucketRequest,
+  ): Promise<string[]> {
+    try {
+      const s3Client = await this.getS3Client('us-east-005');
+      const files: string[] = [];
+      let continuationToken: string | undefined;
+
+      do {
+        const listObjectsV2Command = new ListObjectsV2Command({
+          Bucket: `${request.name}-${request.configId}-${request.configEnv}`,
+          ContinuationToken: continuationToken,
+        });
+        const response = await s3Client.send(listObjectsV2Command);
+        if (response.Contents) {
+          for (const object of response.Contents) {
+            if (object.Key) {
+              files.push(object.Key);
+            }
+          }
+        }
+        continuationToken = response.NextContinuationToken;
+      } while (continuationToken);
+
+      return files;
+    } catch (error) {
+      throw new RpcException({
+        code: status.FAILED_PRECONDITION,
+        message: `Failed to list files: ${error.message} `,
+      });
+    }
+  }
 }

packages/file-service/test/file_bucket.e2e-spec.ts

@@ -267,3 +267,91 @@ describe('File Bucket Creation Tests', () => {
     await createBucketPromise2;
   });
 });
+
+describe('Get All Files Tests', () => {
+  it('Successfully gets all files for a config', async () => {
+    const metadata = {
+      endpoint: baseURL,
+      region: region,
+      credentials: {
+        accessKeyId: accessKeyId as string,
+        secretAccessKey: secretAccessKey as string,
+      },
+    };
+
+    const client = new S3Client(metadata);
+    const command = new DeleteBucketCommand({
+      Bucket: `get-all-files-bucket-${configId}-${configEnv}`,
+    });
+
+    try {
+      await client.send(command);
+    } catch {}
+
+    await new Promise((resolve, reject) => {
+      bucketClient.registerBucket(
+        {
+          name: 'get-all-files-bucket',
+          configId,
+          configEnv,
+          fileProviderName: providerName,
+        },
+        (err: any) => {
+          if (err) return reject(err);
+          resolve(0);
+        },
+      );
+    });
+
+    const getAllFilesPromise = new Promise((resolve, reject) => {
+      bucketClient.getAllFiles(
+        {
+          configId,
+          configEnv,
+        },
+        (err: any, response: any) => {
+          if (err) return reject(err);
+          expect(err).toBeNull();
+          expect(response).toBeDefined();
+          expect(response.files).toBeDefined();
+          resolve(response);
+        },
+      );
+    });
+    await getAllFilesPromise;
+
+    await new Promise((resolve, reject) => {
+      bucketClient.removeBucket(
+        {
+          name: 'get-all-files-bucket',
+          configId,
+          configEnv,
+          fileProviderName: providerName,
+        },
+        (err: any) => {
+          if (err) return reject(err);
+          resolve(0);
+        },
+      );
+    });
+  });
+
+  it('Returns empty files for nonexistent config', async () => {
+    const getAllFilesPromise = new Promise((resolve, reject) => {
+      bucketClient.getAllFiles(
+        {
+          configId: 9999,
+          configEnv: 'nonexistent',
+        },
+        (err: any, response: any) => {
+          if (err) return reject(err);
+          expect(err).toBeNull();
+          expect(response).toBeDefined();
+          expect(response.files ?? []).toEqual([]);
+          resolve(response);
+        },
+      );
+    });
+    await getAllFilesPromise;
+  });
+});

packages/proto/definitions/file_bucket.proto

@@ -16,6 +16,7 @@ service BucketDbService {
 service BucketFileService {
   rpc registerBucket(RegisterBucketRequest) returns (Bucket);
   rpc removeBucket(RemoveBucketRequest) returns (Bucket);
+  rpc getAllFiles(GetAllFilesRequest) returns (GetAllFilesResponse);
 }
 
 message Bucket {
@@ -76,3 +77,17 @@ message RemoveBucketRequest {
   string configEnv = 3;
   string fileProviderName = 4;
 }
+
+message GetAllFilesRequest {
+  int32 configId = 1;
+  string configEnv = 2;
+}
+
+message Files {
+  string bucketName = 1;
+  repeated string files = 2;
+}
+
+message GetAllFilesResponse {
+  repeated Files files = 1;
+}