Skip to content

Commit cec2452

Browse files
kevin-mcgoldrickkevin-mcgoldrick
authored
AWS SDK Consistency Cleanup (#499)
* AWS SDK Consistency Cleanup
1 parent c8f5db9 commit cec2452

14 files changed

Lines changed: 191 additions & 200 deletions

File tree

‎agent/apiharness/src/main/java/com/intuit/tank/harness/TestPlanStarter.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
import com.intuit.tank.harness.logging.LogUtil;
2525
import com.intuit.tank.logging.LogEventType;
2626
import com.intuit.tank.vm.api.enumerated.AgentCommand;
27+
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
28+
import software.amazon.awssdk.core.retry.RetryMode;
2729
import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient;
2830
import software.amazon.awssdk.services.cloudwatch.model.Dimension;
2931
import software.amazon.awssdk.services.cloudwatch.model.MetricDatum;
@@ -74,7 +76,11 @@ public TestPlanStarter(Object httpClient, HDTestPlan plan, int numThreads, Strin
7476
this.rampDelay = calcRampTime();
7577
this.standalone = ((this.numThreads == 1) && (this.agentRunData.getIncrementStrategy().equals(IncrementStrategy.increasing)));
7678
if (!this.standalone) {
77-
this.cloudWatchClient = CloudWatchAsyncClient.builder().build();
79+
this.cloudWatchClient = CloudWatchAsyncClient.builder()
80+
.overrideConfiguration(ClientOverrideConfiguration.builder()
81+
.retryStrategy(RetryMode.ADAPTIVE_V2)
82+
.build())
83+
.build();
7884
this.testPlan = Dimension.builder()
7985
.name("testPlan")
8086
.value(plan.getTestPlanName())
@@ -439,7 +445,11 @@ private void sendCloudWatchMetrics(long activeCount) {
439445
.metricData(datumList)
440446
.build();
441447

442-
cloudWatchClient.putMetricData(request);
448+
cloudWatchClient.putMetricData(request).whenComplete((response, throwable) -> {
449+
if (throwable != null) {
450+
LOG.error(LogUtil.getLogMessage("Failed to push metric data to cloudwatch: " + throwable.getMessage()), throwable);
451+
}
452+
});
443453
send = DateUtils.addSeconds(new Date(), interval); // 15 SECONDS
444454
this.sessionStarts = 0; // reset session starts for next interval
445455
}

‎agent/apiharness/src/test/java/com/intuit/tank/harness/TestPlanStarterTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ public void SetUp() {
3333
_threadGroupMock = mock(ThreadGroup.class);
3434
_agentRunData = mock(AgentRunData.class);
3535
mock_CloudWatchAsyncClient = mockStatic(CloudWatchAsyncClient.class);
36-
when(CloudWatchAsyncClient.builder()).thenReturn(mock(CloudWatchAsyncClientBuilder.class));
36+
when(CloudWatchAsyncClient.builder()).thenReturn(mock(CloudWatchAsyncClientBuilder.class, RETURNS_SELF));
3737
when(_agentRunData.getIncrementStrategy()).thenReturn(IncrementStrategy.increasing);
3838
}
3939

‎api/src/main/java/com/intuit/tank/storage/S3FileStorage.java‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@
2020
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
2121
import software.amazon.awssdk.auth.credentials.AwsCredentials;
2222
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
23+
import software.amazon.awssdk.core.retry.RetryMode;
2324
import software.amazon.awssdk.core.sync.RequestBody;
25+
import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
2426
import software.amazon.awssdk.http.apache.ApacheHttpClient;
2527
import software.amazon.awssdk.http.apache.ProxyConfiguration;
2628
import software.amazon.awssdk.services.s3.S3Client;
@@ -57,7 +59,10 @@ public S3FileStorage(String bucketName, boolean compress) {
5759
TankConfig tankConfig = new TankConfig();
5860
this.encrypt = tankConfig.isS3EncryptionEnabled();
5961
CloudCredentials creds = tankConfig.getVmManagerConfig().getCloudCredentials(CloudProvider.amazon);
60-
S3ClientBuilder s3ClientBuilder = S3Client.builder();
62+
S3ClientBuilder s3ClientBuilder = S3Client.builder()
63+
.overrideConfiguration(ClientOverrideConfiguration.builder()
64+
.retryStrategy(RetryMode.ADAPTIVE_V2)
65+
.build());
6166
if (creds != null && StringUtils.isNotBlank(System.getProperty("http.proxyHost"))) {
6267
try {
6368
URIBuilder uriBuilder = new URIBuilder().setHost(System.getProperty("http.proxyHost"));
@@ -113,7 +118,7 @@ public void storeFileData(FileData fileData, InputStream in) {
113118
}
114119
s3Client.putObject(request.build(), RequestBody.fromInputStream(in, in.available()));
115120
} catch (Exception e) {
116-
LOG.error("Error storing file: " + e, e);
121+
LOG.error("Error storing file: {}", e, e);
117122
throw new RuntimeException(e);
118123
} finally {
119124
IOUtils.closeQuietly(in);
@@ -129,7 +134,6 @@ public InputStream readFileData(FileData fileData) {
129134
}
130135

131136
private void createBucket(String bucketName) {
132-
System.out.println(bucketName);
133137
try {
134138
s3Client.createBucket(CreateBucketRequest.builder().bucket(bucketName).build());
135139
LOG.info("Created bucket {} at now", bucketName);
@@ -166,13 +170,14 @@ public List<FileData> listFileData(String path) {
166170
prefix = prefix + "/";
167171
}
168172
prefix = Strings.CS.removeStart(prefix, "/");
169-
ListObjectsResponse response = s3Client.listObjects(ListObjectsRequest.builder().bucket(bucketName).prefix(prefix).delimiter("/").build());
170-
for (S3Object object : response.contents()) {
173+
ListObjectsV2Request listRequest = ListObjectsV2Request.builder()
174+
.bucket(bucketName).prefix(prefix).delimiter("/").build();
175+
s3Client.listObjectsV2Paginator(listRequest).contents().forEach(object -> {
171176
String fileName = FilenameUtils.getName(FilenameUtils.normalize(object.key()));
172177
if (StringUtils.isNotBlank(fileName)) {
173178
ret.add(new FileData(path, fileName));
174179
}
175-
}
180+
});
176181
} catch (S3Exception e) {
177182
LOG.error("Error Listing Files: {}", e, e);
178183
throw new RuntimeException(e);

‎api/src/test/java/com/intuit/tank/storage/FileStorageFactoryTest.java‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public void init() {
5151
config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME).setLevel(Level.INFO);
5252
ctx.updateLoggers(); // This causes all Loggers to refetch information from their LoggerConfig.
5353
mockStatic_S3Client = mockStatic(S3Client.class);
54-
mock_S3ClientBuilder = mock(S3ClientBuilder.class);
54+
mock_S3ClientBuilder = mock(S3ClientBuilder.class, RETURNS_SELF);
5555
mock_S3Client = mock(S3Client.class);
5656
when(S3Client.builder()).thenReturn(mock_S3ClientBuilder);
5757
when(mock_S3ClientBuilder.build()).thenReturn(mock_S3Client);
@@ -125,7 +125,7 @@ public void testS3FileStorage() throws Exception {
125125

126126
ByteArrayInputStream bis = new ByteArrayInputStream(s.getBytes());
127127
FileStorage storage = FileStorageFactory.getFileStorage("s3:systemstorage/extra/", false);
128-
assertTrue(storage instanceof S3FileStorage);
128+
assertInstanceOf(S3FileStorage.class, storage);
129129
verify(mock_S3Client, times(1))
130130
.createBucket(CreateBucketRequest.builder().bucket("systemstorage").build());
131131

@@ -164,7 +164,7 @@ public void testS3StorageCompressed() throws Exception {
164164
}
165165
assertTrue(storage.exists(fd));
166166
storage.delete(fd);
167-
assertTrue(!storage.exists(fd));
167+
assertFalse(storage.exists(fd));
168168
}
169169

170170
/**
@@ -199,15 +199,15 @@ public void testS3List() throws Exception {
199199
storage.delete(fd2);
200200
assertTrue(listFileData.contains(fd));
201201
assertTrue(listFileData.contains(fd1));
202-
assertTrue(!listFileData.contains(fd2));
203-
204-
assertTrue(!listFileData1.contains(fd));
205-
assertTrue(!listFileData1.contains(fd1));
202+
assertFalse(listFileData.contains(fd2));
203+
204+
assertFalse(listFileData1.contains(fd));
205+
assertFalse(listFileData1.contains(fd1));
206206
assertTrue(listFileData1.contains(fd2));
207-
208-
assertTrue(!storage.exists(fd));
209-
assertTrue(!storage.exists(fd1));
210-
assertTrue(!storage.exists(fd2));
207+
208+
assertFalse(storage.exists(fd));
209+
assertFalse(storage.exists(fd1));
210+
assertFalse(storage.exists(fd2));
211211
}
212212
/**
213213
* set the enviroment variables AWS_SECRET_KEY_ID and AWS_SECRET_KEY before
@@ -243,15 +243,15 @@ public void testFileList() throws Exception {
243243
storage.delete(fd2);
244244
assertTrue(listFileData.contains(fd));
245245
assertTrue(listFileData.contains(fd1));
246-
assertTrue(!listFileData.contains(fd2));
247-
248-
assertTrue(!listFileData1.contains(fd));
249-
assertTrue(!listFileData1.contains(fd1));
246+
assertFalse(listFileData.contains(fd2));
247+
248+
assertFalse(listFileData1.contains(fd));
249+
assertFalse(listFileData1.contains(fd1));
250250
assertTrue(listFileData1.contains(fd2));
251-
252-
assertTrue(!storage.exists(fd));
253-
assertTrue(!storage.exists(fd1));
254-
assertTrue(!storage.exists(fd2));
251+
252+
assertFalse(storage.exists(fd));
253+
assertFalse(storage.exists(fd1));
254+
assertFalse(storage.exists(fd2));
255255
}
256256

257257
private File getFile(String base, FileData fd) {

‎buildspec.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version: 0.2
22

33
env:
44
variables:
5-
MAVEN_OPTS: "-Xms1g -Xmx2g"
5+
MAVEN_OPTS: "-Xms2g -Xmx3g"
66
SKIP_METHODTIMER_TEST: true
77
SKIP_GUI_TEST: true
88

@@ -13,6 +13,7 @@ phases:
1313
build:
1414
commands:
1515
- java -version
16+
- export WATS_PROPERTIES="$CODEBUILD_SRC_DIR/api/src/main/resources"
1617
# Run the complete build with both unit and integration tests
1718
- mvn clean install surefire-report:report -P release
1819
- export PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)

‎pom.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1117,6 +1117,8 @@
11171117
<configuration>
11181118
<!-- Sets the VM argument line used when unit tests are run. -->
11191119
<argLine>${argLine}</argLine>
1120+
<forkCount>0.5C</forkCount>
1121+
<reuseForks>true</reuseForks>
11201122
<includes>
11211123
<include>**/*Test.java</include>
11221124
<include>**/*Spec.groovy</include>

0 commit comments

Comments
 (0)