Skip to content

Commit c2ef7ec

Browse files
nyccreatorasarode06
nyccreator
and
asarode06
authored
Andrey - Get All Files (#250)
Co-authored-by: asarode9 <asarode9@gatech.edu>
1 parent c397ebf commit c2ef7ec

10 files changed

Lines changed: 294 additions & 2 deletions

File tree

‎packages/api-gateway/src/models/file_bucket.dto.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,3 +109,11 @@ export class FileBucket implements FileBucketProto.Bucket {
109109
this.FileServiceFile = fileBucket.FileServiceFile ?? [];
110110
}
111111
}
112+
113+
export class BucketWithFiles {
114+
[bucketName: string]: string[];
115+
116+
constructor(entry: FileBucketProto.Files) {
117+
this[entry.bucketName] = entry.files;
118+
}
119+
}

‎packages/api-gateway/src/modules/file_bucket/file_bucket.controller.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import { AuthCommonProto, FileBucketProto } from 'juno-proto';
2323
import { lastValueFrom } from 'rxjs';
2424
import { ApiKey } from 'src/decorators/api_key.decorator';
2525
import {
26+
BucketWithFiles,
2627
DeleteFileBucketModel,
2728
FileBucket,
2829
RegisterFileBucketModel,
@@ -132,4 +133,33 @@ export class FileBucketController implements OnModuleInit {
132133

133134
return bucketsData.buckets?.map((bucket) => new FileBucket(bucket)) ?? [];
134135
}
136+
137+
@Get('all/:configId')
138+
@ApiOperation({ summary: 'Get all files from all buckets for a config.' })
139+
@ApiBadRequestResponse({ description: 'Parameters are invalid' })
140+
@ApiUnauthorizedResponse({ description: 'Unauthorized' })
141+
@ApiOkResponse({
142+
description: 'Returned all files grouped by bucket name',
143+
type: [BucketWithFiles],
144+
})
145+
async getAllFiles(
146+
@ApiKey() apiKey: AuthCommonProto.ApiKey,
147+
@Param('configId') configId: string,
148+
): Promise<BucketWithFiles[]> {
149+
const id = parseInt(configId);
150+
if (Number.isNaN(id) || id < 0) {
151+
throw new BadRequestException(
152+
'Id must be an int greater than or equal to 0',
153+
);
154+
}
155+
156+
const grpcResponse = this.fileBucketService.getAllFiles({
157+
configId: id,
158+
configEnv: apiKey.environment,
159+
});
160+
161+
const data = await lastValueFrom(grpcResponse);
162+
163+
return (data.files ?? []).map((entry) => new BucketWithFiles(entry));
164+
}
135165
}

‎packages/api-gateway/src/modules/file_bucket/file_bucket.module.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ import {
7272
})
7373
export class FileBucketModule implements NestModule {
7474
configure(consumer: MiddlewareConsumer) {
75-
consumer.apply(ApiKeyMiddleware).forRoutes('/file/bucket', 'file/bucket/*');
75+
consumer
76+
.apply(ApiKeyMiddleware)
77+
.forRoutes('/file/bucket', 'file/bucket/*', '/file/all/*');
7678
}
7779
}

‎packages/api-gateway/test/file_bucket.e2e-spec.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,4 +230,22 @@ describe('File Bucket Routes', () => {
230230

231231
await client.send(command);
232232
});
233+
234+
it('Gets all files successfully', async () => {
235+
await request(app.getHttpServer())
236+
.get('/file/all/0')
237+
.set('Authorization', 'Bearer ' + apiKey)
238+
.expect(200);
239+
});
240+
241+
it('Unsuccessfully gets all files due to invalid configId', async () => {
242+
await request(app.getHttpServer())
243+
.get('/file/all/invalid')
244+
.set('Authorization', 'Bearer ' + apiKey)
245+
.expect(400);
246+
});
247+
248+
it('Unsuccessfully gets all files due to missing authorization', async () => {
249+
await request(app.getHttpServer()).get('/file/all/0').expect(401);
250+
});
233251
});

‎packages/file-service/src/modules/file_bucket/azure_handler.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,4 +68,25 @@ export class AzureBucketHandler {
6868
});
6969
}
7070
}
71+
72+
async listFiles(
73+
request: FileBucketProto.GetBucketRequest,
74+
): Promise<string[]> {
75+
try {
76+
const blobClient = await this.getBlobServiceClient();
77+
const containerClient = blobClient.getContainerClient(
78+
`${request.name}-${request.configId}-${request.configEnv}`,
79+
);
80+
const files: string[] = [];
81+
for await (const blob of containerClient.listBlobsFlat()) {
82+
files.push(blob.name);
83+
}
84+
return files;
85+
} catch (error) {
86+
throw new RpcException({
87+
code: status.FAILED_PRECONDITION,
88+
message: `Failed to list files: ${error.message} `,
89+
});
90+
}
91+
}
7192
}

‎packages/file-service/src/modules/file_bucket/file_bucket.controller.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,10 @@ export class FileBucketController
2020
): Promise<FileBucketProto.Bucket> {
2121
return await this.fileBucketService.removeBucket(request);
2222
}
23+
24+
async getAllFiles(
25+
request: FileBucketProto.GetAllFilesRequest,
26+
): Promise<FileBucketProto.GetAllFilesResponse> {
27+
return await this.fileBucketService.getAllFiles(request);
28+
}
2329
}

‎packages/file-service/src/modules/file_bucket/file_bucket.service.ts‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { status } from '@grpc/grpc-js';
2-
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
2+
import { Inject, Injectable, Logger, OnModuleInit } from '@nestjs/common';
33
import { ClientGrpc, RpcException } from '@nestjs/microservices';
44
import { FileBucketProto, FileProviderProto } from 'juno-proto';
55
import { lastValueFrom } from 'rxjs';
@@ -8,6 +8,7 @@ import { S3BucketHandler } from './s3_handler';
88

99
@Injectable()
1010
export class FileBucketService implements OnModuleInit {
11+
private readonly logger = new Logger(FileBucketService.name);
1112
private fileBucketDBService: FileBucketProto.BucketDbServiceClient;
1213
private fileProviderDBService: FileProviderProto.FileProviderDbServiceClient;
1314

@@ -134,4 +135,73 @@ export class FileBucketService implements OnModuleInit {
134135
});
135136
}
136137
}
138+
139+
async getAllFiles(
140+
request: FileBucketProto.GetAllFilesRequest,
141+
): Promise<FileBucketProto.GetAllFilesResponse> {
142+
try {
143+
const buckets = await lastValueFrom(
144+
this.fileBucketDBService.getBucketsByConfigIdAndEnv({
145+
configId: request.configId,
146+
configEnv: request.configEnv,
147+
}),
148+
);
149+
150+
const results = await Promise.all(
151+
(buckets.buckets ?? []).map(async (bucket) => {
152+
try {
153+
const provider = await this.getProvider(bucket.fileProviderName);
154+
const bucketRequest: FileBucketProto.GetBucketRequest = {
155+
name: bucket.name,
156+
configId: bucket.configId,
157+
configEnv: bucket.configEnv,
158+
};
159+
160+
let fileList: string[];
161+
switch (provider.providerType) {
162+
case FileProviderProto.ProviderType.S3: {
163+
const handler = new S3BucketHandler(provider);
164+
fileList = await handler.listFiles(bucketRequest);
165+
break;
166+
}
167+
case FileProviderProto.ProviderType.AZURE: {
168+
const handler = new AzureBucketHandler(provider);
169+
fileList = await handler.listFiles(bucketRequest);
170+
break;
171+
}
172+
default:
173+
throw new RpcException({
174+
code: status.FAILED_PRECONDITION,
175+
message: `Unsupported provider type: ${provider.providerType} `,
176+
});
177+
}
178+
179+
return {
180+
bucketName: bucket.name,
181+
files: fileList,
182+
};
183+
} catch (error) {
184+
this.logger.error(
185+
`Failed to list files for bucket "${bucket.name}": ${error?.message}`,
186+
);
187+
return null;
188+
}
189+
}),
190+
);
191+
192+
const files = results.filter(
193+
(result): result is FileBucketProto.Files => result !== null,
194+
);
195+
196+
return { files };
197+
} catch (error) {
198+
if (error instanceof RpcException) {
199+
throw error;
200+
}
201+
throw new RpcException({
202+
code: status.FAILED_PRECONDITION,
203+
message: `Failed to get all files: ${error.message} `,
204+
});
205+
}
206+
}
137207
}

‎packages/file-service/src/modules/file_bucket/s3_handler.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import {
22
CreateBucketCommand,
33
DeleteBucketCommand,
4+
ListObjectsV2Command,
45
S3Client,
56
} from '@aws-sdk/client-s3';
67
import { status } from '@grpc/grpc-js';
@@ -67,4 +68,37 @@ export class S3BucketHandler {
6768
});
6869
}
6970
}
71+
72+
async listFiles(
73+
request: FileBucketProto.GetBucketRequest,
74+
): Promise<string[]> {
75+
try {
76+
const s3Client = await this.getS3Client('us-east-005');
77+
const files: string[] = [];
78+
let continuationToken: string | undefined;
79+
80+
do {
81+
const listObjectsV2Command = new ListObjectsV2Command({
82+
Bucket: `${request.name}-${request.configId}-${request.configEnv}`,
83+
ContinuationToken: continuationToken,
84+
});
85+
const response = await s3Client.send(listObjectsV2Command);
86+
if (response.Contents) {
87+
for (const object of response.Contents) {
88+
if (object.Key) {
89+
files.push(object.Key);
90+
}
91+
}
92+
}
93+
continuationToken = response.NextContinuationToken;
94+
} while (continuationToken);
95+
96+
return files;
97+
} catch (error) {
98+
throw new RpcException({
99+
code: status.FAILED_PRECONDITION,
100+
message: `Failed to list files: ${error.message} `,
101+
});
102+
}
103+
}
70104
}

‎packages/file-service/test/file_bucket.e2e-spec.ts‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,3 +267,91 @@ describe('File Bucket Creation Tests', () => {
267267
await createBucketPromise2;
268268
});
269269
});
270+
271+
describe('Get All Files Tests', () => {
272+
it('Successfully gets all files for a config', async () => {
273+
const metadata = {
274+
endpoint: baseURL,
275+
region: region,
276+
credentials: {
277+
accessKeyId: accessKeyId as string,
278+
secretAccessKey: secretAccessKey as string,
279+
},
280+
};
281+
282+
const client = new S3Client(metadata);
283+
const command = new DeleteBucketCommand({
284+
Bucket: `get-all-files-bucket-${configId}-${configEnv}`,
285+
});
286+
287+
try {
288+
await client.send(command);
289+
} catch {}
290+
291+
await new Promise((resolve, reject) => {
292+
bucketClient.registerBucket(
293+
{
294+
name: 'get-all-files-bucket',
295+
configId,
296+
configEnv,
297+
fileProviderName: providerName,
298+
},
299+
(err: any) => {
300+
if (err) return reject(err);
301+
resolve(0);
302+
},
303+
);
304+
});
305+
306+
const getAllFilesPromise = new Promise((resolve, reject) => {
307+
bucketClient.getAllFiles(
308+
{
309+
configId,
310+
configEnv,
311+
},
312+
(err: any, response: any) => {
313+
if (err) return reject(err);
314+
expect(err).toBeNull();
315+
expect(response).toBeDefined();
316+
expect(response.files).toBeDefined();
317+
resolve(response);
318+
},
319+
);
320+
});
321+
await getAllFilesPromise;
322+
323+
await new Promise((resolve, reject) => {
324+
bucketClient.removeBucket(
325+
{
326+
name: 'get-all-files-bucket',
327+
configId,
328+
configEnv,
329+
fileProviderName: providerName,
330+
},
331+
(err: any) => {
332+
if (err) return reject(err);
333+
resolve(0);
334+
},
335+
);
336+
});
337+
});
338+
339+
it('Returns empty files for nonexistent config', async () => {
340+
const getAllFilesPromise = new Promise((resolve, reject) => {
341+
bucketClient.getAllFiles(
342+
{
343+
configId: 9999,
344+
configEnv: 'nonexistent',
345+
},
346+
(err: any, response: any) => {
347+
if (err) return reject(err);
348+
expect(err).toBeNull();
349+
expect(response).toBeDefined();
350+
expect(response.files ?? []).toEqual([]);
351+
resolve(response);
352+
},
353+
);
354+
});
355+
await getAllFilesPromise;
356+
});
357+
});

‎packages/proto/definitions/file_bucket.proto‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ service BucketDbService {
1616
service BucketFileService {
1717
rpc registerBucket(RegisterBucketRequest) returns (Bucket);
1818
rpc removeBucket(RemoveBucketRequest) returns (Bucket);
19+
rpc getAllFiles(GetAllFilesRequest) returns (GetAllFilesResponse);
1920
}
2021

2122
message Bucket {
@@ -76,3 +77,17 @@ message RemoveBucketRequest {
7677
string configEnv = 3;
7778
string fileProviderName = 4;
7879
}
80+
81+
message GetAllFilesRequest {
82+
int32 configId = 1;
83+
string configEnv = 2;
84+
}
85+
86+
message Files {
87+
string bucketName = 1;
88+
repeated string files = 2;
89+
}
90+
91+
message GetAllFilesResponse {
92+
repeated Files files = 1;
93+
}

0 commit comments

Comments
 (0)