file services bucket tagging + file service config setup (#200)

Co-authored-by: Long Lam <llam36@gatech.edu>

Author: tmthecoder

packages/api-gateway/src/models/file_config.dto.ts

@@ -53,3 +53,15 @@ export class FileConfigResponse {
     this.buckets = fileConfig.buckets;
   }
 }
+
+export class SetupFileServiceResponse {
+  @ApiProperty({
+    type: 'boolean',
+    description: 'Whether file services were successfully setup',
+  })
+  success: boolean;
+
+  constructor(res: FileConfigProto.SetupResponse) {
+    this.success = res.success;
+  }
+}

packages/api-gateway/src/modules/file_config/file_config.controller.ts

@@ -5,11 +5,13 @@ import {
   Inject,
   OnModuleInit,
   Param,
+  Post,
 } from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
 import {
   ApiBadRequestResponse,
   ApiBearerAuth,
+  ApiCreatedResponse,
   ApiNotFoundResponse,
   ApiOkResponse,
   ApiOperation,
@@ -19,26 +21,39 @@ import {
 import { AuthCommonProto, FileConfigProto } from 'juno-proto';
 import { lastValueFrom } from 'rxjs';
 import { ApiKey } from 'src/decorators/api_key.decorator';
-import { FileConfigResponse } from 'src/models/file_config.dto';
+import {
+  FileConfigResponse,
+  SetupFileServiceResponse,
+} from 'src/models/file_config.dto';
 
-const { FILE_SERVICE_CONFIG_DB_SERVICE_NAME } = FileConfigProto;
+const {
+  FILE_SERVICE_CONFIG_DB_SERVICE_NAME,
+  FILE_SERVICE_CONFIG_SERVICE_NAME,
+} = FileConfigProto;
 
 @ApiBearerAuth('api_key')
 @ApiTags('file_config')
 @Controller('file')
 export class FileConfigController implements OnModuleInit {
+  private fileConfigService: FileConfigProto.FileServiceConfigServiceClient;
   private fileConfigDBService: FileConfigProto.FileServiceConfigDbServiceClient;
 
   constructor(
     @Inject(FILE_SERVICE_CONFIG_DB_SERVICE_NAME)
+    private fileConfigDbClient: ClientGrpc,
+    @Inject(FILE_SERVICE_CONFIG_SERVICE_NAME)
     private fileConfigClient: ClientGrpc,
   ) {}
 
   onModuleInit() {
     this.fileConfigDBService =
-      this.fileConfigClient.getService<FileConfigProto.FileServiceConfigDbServiceClient>(
+      this.fileConfigDbClient.getService<FileConfigProto.FileServiceConfigDbServiceClient>(
         FILE_SERVICE_CONFIG_DB_SERVICE_NAME,
       );
+    this.fileConfigService =
+      this.fileConfigClient.getService<FileConfigProto.FileServiceConfigServiceClient>(
+        FILE_SERVICE_CONFIG_SERVICE_NAME,
+      );
   }
 
   @Get('config/:projectId')
@@ -75,4 +90,27 @@ export class FileConfigController implements OnModuleInit {
 
     return new FileConfigResponse(await lastValueFrom(config));
   }
+
+  @ApiOperation({
+    summary: 'Sets up file services for the Project/Environment',
+  })
+  @ApiCreatedResponse({
+    description: 'File Services setup successfully',
+    type: SetupFileServiceResponse,
+  })
+  @ApiUnauthorizedResponse({ description: 'Unauthorized' })
+  @ApiBadRequestResponse({ description: 'Bad Request' })
+  @Post('config/setup')
+  async setup(
+    @ApiKey() apiKey: AuthCommonProto.ApiKey,
+  ): Promise<SetupFileServiceResponse> {
+    const setupResponse = await lastValueFrom(
+      this.fileConfigService.setup({
+        projectId: apiKey.project.id,
+        environment: apiKey.environment,
+      }),
+    );
+
+    return new SetupFileServiceResponse(setupResponse);
+  }
 }

packages/api-gateway/src/modules/file_config/file_config.module.ts

@@ -15,7 +15,10 @@ import {
   API_KEY_SERVICE_NAME,
   JUNO_API_KEY_PACKAGE_NAME,
 } from 'juno-proto/dist/gen/api_key';
-import { FILE_SERVICE_CONFIG_DB_SERVICE_NAME } from 'juno-proto/dist/gen/file_config';
+import {
+  FILE_SERVICE_CONFIG_DB_SERVICE_NAME,
+  FILE_SERVICE_CONFIG_SERVICE_NAME,
+} from 'juno-proto/dist/gen/file_config';
 import {
   JWT_SERVICE_NAME,
   JUNO_JWT_PACKAGE_NAME,
@@ -45,6 +48,15 @@ import {
           protoPath: FileConfigProtoFile,
         },
       },
+      {
+        name: FILE_SERVICE_CONFIG_SERVICE_NAME,
+        transport: Transport.GRPC,
+        options: {
+          url: process.env.FILE_SERVICE_ADDR,
+          package: FileConfigProto.JUNO_FILE_SERVICE_CONFIG_PACKAGE_NAME,
+          protoPath: FileConfigProtoFile,
+        },
+      },
       {
         name: JWT_SERVICE_NAME,
         transport: Transport.GRPC,

packages/api-gateway/test/file_bucket.e2e-spec.ts

@@ -143,7 +143,9 @@ describe('File Bucket Routes', () => {
       },
     };
     const client = new S3Client(metadata);
-    const command = new DeleteBucketCommand({ Bucket: uniqueBucketName });
+    const command = new DeleteBucketCommand({
+      Bucket: `${uniqueBucketName}-0-prod`,
+    });
 
     await client.send(command);
   });
@@ -223,7 +225,7 @@ describe('File Bucket Routes', () => {
     };
     const client = new S3Client(metadata);
     const command = new DeleteBucketCommand({
-      Bucket: uniqueBucketName + '-duplicate',
+      Bucket: uniqueBucketName + '-duplicate-0-prod',
     });
 
     await client.send(command);

packages/api-gateway/test/file_config.e2e-spec.ts

@@ -1,16 +1,16 @@
-import { Test, TestingModule } from '@nestjs/testing';
+import * as GRPC from '@grpc/grpc-js';
+import * as ProtoLoader from '@grpc/proto-loader';
 import {
   ClassSerializerInterceptor,
   INestApplication,
   ValidationPipe,
 } from '@nestjs/common';
-import { AppModule } from './../src/app.module';
 import { Reflector } from '@nestjs/core';
-import * as request from 'supertest';
+import { Test, TestingModule } from '@nestjs/testing';
 import { ResetProtoFile } from 'juno-proto';
-import * as GRPC from '@grpc/grpc-js';
-import * as ProtoLoader from '@grpc/proto-loader';
 import { RpcExceptionFilter } from 'src/rpc_exception_filter';
+import * as request from 'supertest';
+import { AppModule } from './../src/app.module';
 
 let app: INestApplication;
 const ADMIN_EMAIL = 'test-superadmin@test.com';
@@ -101,4 +101,17 @@ describe('File Config Routes', () => {
       .set('Authorization', 'Bearer ' + apiKey)
       .expect(404);
   });
+
+  it('Should setup file services for a valid project + api key', async () => {
+    return await request(app.getHttpServer())
+      .post('/file/config/setup')
+      .set('Authorization', 'Bearer ' + apiKey)
+      .expect(201);
+  });
+
+  it('Should throw a 401 status error when setup file services without valid credentials', async () => {
+    return await request(app.getHttpServer())
+      .post('/file/config/setup')
+      .expect(401);
+  });
 });

packages/db-service/src/modules/file_config/file_config.controller.ts

@@ -1,9 +1,9 @@
+import { status } from '@grpc/grpc-js';
 import { Controller } from '@nestjs/common';
-import { FileServiceConfigService } from './file_config.service';
 import { RpcException } from '@nestjs/microservices';
-import { status } from '@grpc/grpc-js';
-import { FileConfigProto } from 'juno-proto';
 import { Prisma } from '@prisma/client';
+import { FileConfigProto } from 'juno-proto';
+import { FileServiceConfigService } from './file_config.service';
 
 @Controller()
 @FileConfigProto.FileServiceConfigDbServiceControllerMethods()
@@ -26,16 +26,15 @@ export class FileConfigController
         environment,
       });
       if (existingConfig) {
-        throw new RpcException({
-          code: status.ALREADY_EXISTS,
-          message: 'Config already exists',
-        });
+        return {
+          id: existingConfig.id,
+          environment: existingConfig.environment,
+          files: [],
+          buckets: [],
+        };
       }
-    } catch (error) {
-      throw new RpcException({
-        code: status.ALREADY_EXISTS,
-        message: 'Config already exists',
-      });
+    } catch {
+      // Intentionally ignoring getConfig error for now
     }
 
     const config = await this.fileServiceConfigService.createConfig(request);

packages/db-service/test/file_config.e2e-spec.ts

@@ -1,15 +1,15 @@
-import { Test, TestingModule } from '@nestjs/testing';
-import { INestMicroservice } from '@nestjs/common';
-import { AppModule } from '../src/app.module';
-import * as ProtoLoader from '@grpc/proto-loader';
 import * as GRPC from '@grpc/grpc-js';
+import * as ProtoLoader from '@grpc/proto-loader';
+import { INestMicroservice } from '@nestjs/common';
 import { MicroserviceOptions, Transport } from '@nestjs/microservices';
+import { Test, TestingModule } from '@nestjs/testing';
 import {
   FileConfigProto,
   FileConfigProtoFile,
   ResetProto,
   ResetProtoFile,
 } from 'juno-proto';
+import { AppModule } from '../src/app.module';
 
 let app: INestMicroservice;
 let createdConfigId: number;
@@ -135,8 +135,8 @@ describe('File Service Config Tests', () => {
   });
 
   it('creates a duplicate file service config', async () => {
-    try {
-      await new Promise((resolve, reject) => {
+    const response: FileConfigProto.FileServiceConfig = await new Promise(
+      (resolve, reject) => {
         configClient.createConfig(
           {
             projectId: 0,
@@ -152,10 +152,9 @@ describe('File Service Config Tests', () => {
             }
           },
         );
-      });
-    } catch (err) {
-      expect(err).toBeDefined();
-    }
+      },
+    );
+    expect(response).toHaveProperty('id');
   });
 
   it('deletes a config', async () => {

packages/file-service/src/app.module.ts

@@ -1,19 +1,21 @@
 import { Module } from '@nestjs/common';
+import { SentryModule } from '@sentry/nestjs/setup';
 import { AppController } from './app.controller';
 import { AppService } from './app.service';
-import { SentryModule } from '@sentry/nestjs/setup';
-import { HealthModule } from './modules/health/health.module';
-import { FileUploadModule } from './modules/file_upload/file_upload.module';
-import { FileProviderModule } from './modules/file_provider/file_provider.module';
 import { FileBucketModule } from './modules/file_bucket/file_bucket.module';
+import { FileConfigModule } from './modules/file_config/file_config.module';
+import { FileProviderModule } from './modules/file_provider/file_provider.module';
+import { FileTransferModule } from './modules/file_transfer/file_transfer.module';
+import { HealthModule } from './modules/health/health.module';
 
 @Module({
   imports: [
     SentryModule.forRoot(),
     HealthModule,
-    FileUploadModule,
+    FileTransferModule,
     FileProviderModule,
     FileBucketModule,
+    FileConfigModule,
   ],
   controllers: [AppController],
   providers: [AppService],

packages/file-service/src/modules/file_bucket/azure_handler.ts

@@ -46,7 +46,11 @@ export class AzureBucketHandler {
   ): Promise<FileBucketProto.Bucket> {
     try {
       const blobClient = await this.getBlobServiceClient();
-      await blobClient.getContainerClient(request.name).create();
+      await blobClient
+        .getContainerClient(
+          `${request.name}-${request.configId}-${request.configEnv}`,
+        )
+        .create();
 
       const dbBucket = await lastValueFrom(
         this.fileDBService.createBucket(request),
@@ -72,7 +76,11 @@ export class AzureBucketHandler {
         }),
       );
       const blobClient = await this.getBlobServiceClient();
-      await blobClient.getContainerClient(request.name).delete();
+      await blobClient
+        .getContainerClient(
+          `${request.name}-${request.configId}-${request.configEnv}`,
+        )
+        .delete();
       return bucket;
     } catch (error) {
       throw new RpcException({

packages/file-service/src/modules/file_bucket/s3_handler.ts

@@ -36,7 +36,7 @@ export class S3BucketHandler {
     try {
       const s3Client = await this.getS3Client('us-east-005');
       const createBucketCommand = new CreateBucketCommand({
-        Bucket: request.name,
+        Bucket: `${request.name}-${request.configId}-${request.configEnv}`,
       });
       await s3Client.send(createBucketCommand);
 
@@ -64,7 +64,7 @@ export class S3BucketHandler {
     try {
       const s3Client = await this.getS3Client('us-east-005');
       const deleteBucketCommand = new DeleteBucketCommand({
-        Bucket: request.name,
+        Bucket: `${request.name}-${request.configId}-${request.configEnv}`,
       });
       await s3Client.send(deleteBucketCommand);
       const bucket = await lastValueFrom(