|
| 1 | +/* |
| 2 | + * Copyright 2026 Google LLC |
| 3 | + * |
| 4 | + * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | + * you may not use this file except in compliance with the License. |
| 6 | + * You may obtain a copy of the License at |
| 7 | + * |
| 8 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | + * |
| 10 | + * Unless required by applicable law or agreed to in writing, software |
| 11 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | + * See the License for the specific language governing permissions and |
| 14 | + * limitations under the License. |
| 15 | + */ |
| 16 | + |
| 17 | +package com.example.storage.object; |
| 18 | + |
| 19 | +// [START storage_optimize_write_latency_pool] |
| 20 | + |
| 21 | +import com.google.cloud.storage.BlobAppendableUpload.AppendableUploadWriteableByteChannel; |
| 22 | +import com.google.cloud.storage.BlobAppendableUploadConfig; |
| 23 | +import com.google.cloud.storage.BlobAppendableUploadConfig.CloseAction; |
| 24 | +import com.google.cloud.storage.BlobId; |
| 25 | +import com.google.cloud.storage.BlobInfo; |
| 26 | +import com.google.cloud.storage.BlobReadSession; |
| 27 | +import com.google.cloud.storage.RangeSpec; |
| 28 | +import com.google.cloud.storage.ReadProjectionConfigs; |
| 29 | +import com.google.cloud.storage.Storage; |
| 30 | +import com.google.cloud.storage.StorageOptions; |
| 31 | +import java.nio.ByteBuffer; |
| 32 | +import java.nio.charset.StandardCharsets; |
| 33 | +import java.util.Queue; |
| 34 | +import java.util.concurrent.ConcurrentLinkedQueue; |
| 35 | +import java.util.concurrent.ExecutorService; |
| 36 | +import java.util.concurrent.Executors; |
| 37 | +import java.util.concurrent.Future; |
| 38 | +import java.util.concurrent.TimeUnit; |
| 39 | + |
| 40 | +public class OptimizeWriteLatencyPool { |
| 41 | + public static void optimizeWriteLatencyPool(String bucketName, String keyPrefix) |
| 42 | + throws Exception { |
| 43 | + // The ID of your GCS zonal bucket |
| 44 | + // String bucketName = "your-unique-bucket-name"; |
| 45 | + |
| 46 | + // The prefix for your pooled GCS objects |
| 47 | + // String keyPrefix = "pooled-object"; |
| 48 | + |
| 49 | + int poolSize = 3; |
| 50 | + String nextObjectName = keyPrefix + "_" + poolSize; |
| 51 | + byte[] payload = "0123456789".getBytes(StandardCharsets.UTF_8); |
| 52 | + |
| 53 | + try (Storage storage = StorageOptions.grpc().build().getService()) { |
| 54 | + BlobAppendableUploadConfig config = |
| 55 | + BlobAppendableUploadConfig.of().withCloseAction(CloseAction.CLOSE_WITHOUT_FINALIZING); |
| 56 | + Queue<AppendableUploadWriteableByteChannel> pool = new ConcurrentLinkedQueue<>(); |
| 57 | + ExecutorService executor = Executors.newSingleThreadExecutor(); |
| 58 | + try { |
| 59 | + // 1. Init pool: Sized to ensure pre-warmed channels are always available. |
| 60 | + for (int i = 0; i < poolSize; i++) { |
| 61 | + BlobInfo info = BlobInfo.newBuilder(bucketName, keyPrefix + "_" + i).build(); |
| 62 | + // open() establishes the stream and creates the 0-byte object in the background. |
| 63 | + pool.add( |
| 64 | + storage |
| 65 | + .blobAppendableUpload(info, config, Storage.BlobWriteOption.doesNotExist()) |
| 66 | + .open()); |
| 67 | + } |
| 68 | + |
| 69 | + // 2. Write: Pop a pre-warmed writer and commit with flush() (~1-2 ms) |
| 70 | + // instead of blocking on close(). |
| 71 | + AppendableUploadWriteableByteChannel channel = pool.poll(); |
| 72 | + if (channel == null) { |
| 73 | + throw new IllegalStateException("Writer pool is empty"); |
| 74 | + } |
| 75 | + try { |
| 76 | + channel.write(ByteBuffer.wrap(payload)); |
| 77 | + channel.flush(); |
| 78 | + } catch (Exception e) { |
| 79 | + try { |
| 80 | + channel.closeWithoutFinalizing(); |
| 81 | + } catch (Exception closeException) { |
| 82 | + e.addSuppressed(closeException); |
| 83 | + } |
| 84 | + throw e; |
| 85 | + } |
| 86 | + |
| 87 | + // 3. Pool maintenance (run asynchronously off the critical write path): |
| 88 | + // Close the used channel without finalizing and refill the pool. |
| 89 | + Future<Void> maintenanceFuture = |
| 90 | + executor.submit( |
| 91 | + () -> { |
| 92 | + channel.closeWithoutFinalizing(); |
| 93 | + BlobInfo nextInfo = BlobInfo.newBuilder(bucketName, nextObjectName).build(); |
| 94 | + pool.add( |
| 95 | + storage |
| 96 | + .blobAppendableUpload( |
| 97 | + nextInfo, config, Storage.BlobWriteOption.doesNotExist()) |
| 98 | + .open()); |
| 99 | + return null; |
| 100 | + }); |
| 101 | + |
| 102 | + // 4. Read: Unfinalized objects are readable after flush(). |
| 103 | + try (BlobReadSession readSession = |
| 104 | + storage |
| 105 | + .blobReadSession(BlobId.of(bucketName, keyPrefix + "_0")) |
| 106 | + .get(10, TimeUnit.SECONDS)) { |
| 107 | + byte[] bytes = |
| 108 | + readSession |
| 109 | + .readAs( |
| 110 | + ReadProjectionConfigs.asFutureBytes() |
| 111 | + .withRangeSpec(RangeSpec.of(0, payload.length))) |
| 112 | + .get(); |
| 113 | + System.out.printf( |
| 114 | + "Read unfinalized object %s_0: %s%n", |
| 115 | + keyPrefix, new String(bytes, StandardCharsets.UTF_8)); |
| 116 | + } |
| 117 | + |
| 118 | + maintenanceFuture.get(10, TimeUnit.SECONDS); |
| 119 | + } finally { |
| 120 | + // Wait for in-flight maintenance so it can't add a channel after the pool is drained. |
| 121 | + executor.shutdown(); |
| 122 | + executor.awaitTermination(10, TimeUnit.SECONDS); |
| 123 | + // Runs before the Storage client is closed by the outer try-with-resources. |
| 124 | + AppendableUploadWriteableByteChannel remaining; |
| 125 | + while ((remaining = pool.poll()) != null) { |
| 126 | + try { |
| 127 | + remaining.closeWithoutFinalizing(); |
| 128 | + } catch (Exception e) { |
| 129 | + // Ignore so the remaining channels are still closed. |
| 130 | + } |
| 131 | + } |
| 132 | + } |
| 133 | + } |
| 134 | + } |
| 135 | +} |
| 136 | + |
| 137 | +// [END storage_optimize_write_latency_pool] |
0 commit comments