Skip to content

Commit 415cfdf

Browse files
Fix delaying queue executor ownership and shutdown (#4977)
Co-authored-by: brendandburns <5751682+brendandburns@users.noreply.github.com>
1 parent 34d3caf commit 415cfdf

6 files changed

Lines changed: 237 additions & 11 deletions

File tree

‎extended/src/main/java/io/kubernetes/client/extended/controller/builder/DefaultControllerBuilder.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ public class DefaultControllerBuilder {
4646
DefaultControllerBuilder() {
4747
this.workerCount = Constants.DEFAULT_WORKER_COUNT;
4848
this.controllerName = "default-controller";
49-
this.workQueue = new DefaultRateLimitingQueue<>(Executors.newSingleThreadExecutor());
49+
this.workQueue = new DefaultRateLimitingQueue<>();
5050
this.readyFuncs = new ArrayList<>();
5151
}
5252

‎extended/src/main/java/io/kubernetes/client/extended/workqueue/DefaultDelayingQueue.java‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.util.concurrent.Delayed;
2222
import java.util.concurrent.ExecutorService;
2323
import java.util.concurrent.Executors;
24+
import java.util.concurrent.Future;
2425
import java.util.concurrent.LinkedBlockingQueue;
2526
import java.util.concurrent.TimeUnit;
2627
import java.util.function.Supplier;
@@ -34,21 +35,42 @@ public class DefaultDelayingQueue<T> extends DefaultWorkQueue<T> implements Dela
3435
private ConcurrentMap<T, WaitForEntry<T>> waitingEntryByData;
3536
protected BlockingQueue<WaitForEntry<T>> waitingForAddQueue;
3637
private Supplier<Long> timeSource;
38+
private final ExecutorService ownedWaitingWorker;
39+
private final Future<?> waitingTask;
3740

3841
private static Long now() {
3942
return System.nanoTime() / 1000000;
4043
}
4144

45+
/**
46+
* Creates a queue using a caller-owned executor. Shutdown cancels the waiting task but does not
47+
* shut down the executor.
48+
*/
4249
public DefaultDelayingQueue(ExecutorService waitingWorker) {
50+
this(waitingWorker, false);
51+
}
52+
53+
private DefaultDelayingQueue(ExecutorService waitingWorker, boolean ownsWaitingWorker) {
54+
this.ownedWaitingWorker = ownsWaitingWorker ? waitingWorker : null;
4355
this.timeSource = DefaultDelayingQueue::now;
4456
this.delayQueue = new DelayQueue<>();
4557
this.waitingEntryByData = new ConcurrentHashMap<>();
4658
this.waitingForAddQueue = new LinkedBlockingQueue<>(1000);
47-
waitingWorker.submit(this::waitingLoop);
59+
this.waitingTask = waitingWorker.submit(this::waitingLoop);
4860
}
4961

62+
/** Creates a queue that shuts down its internally owned executor on shutdown. */
5063
public DefaultDelayingQueue() {
51-
this(Executors.newSingleThreadExecutor());
64+
this(Executors.newSingleThreadExecutor(), true);
65+
}
66+
67+
@Override
68+
public void shutDown() {
69+
super.shutDown();
70+
waitingTask.cancel(true);
71+
if (ownedWaitingWorker != null) {
72+
ownedWaitingWorker.shutdown();
73+
}
5274
}
5375

5476
public void addAfter(T item, Duration duration) {

‎extended/src/main/java/io/kubernetes/client/extended/workqueue/DefaultRateLimitingQueue.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,22 +15,25 @@
1515
import io.kubernetes.client.extended.workqueue.ratelimiter.DefaultControllerRateLimiter;
1616
import io.kubernetes.client.extended.workqueue.ratelimiter.RateLimiter;
1717
import java.util.concurrent.ExecutorService;
18-
import java.util.concurrent.Executors;
1918

2019
/** The default rate limiting queue implementation. */
2120
public class DefaultRateLimitingQueue<T> extends DefaultDelayingQueue<T>
2221
implements RateLimitingQueue<T> {
2322

2423
private RateLimiter<T> rateLimiter;
2524

25+
/** Creates a queue that shuts down its internally owned executor on shutdown. */
2626
public DefaultRateLimitingQueue() {
27-
this(Executors.newSingleThreadExecutor());
27+
super();
28+
this.rateLimiter = new DefaultControllerRateLimiter<>();
2829
}
2930

31+
/** Creates a queue using a caller-owned executor that is not shut down by this queue. */
3032
public DefaultRateLimitingQueue(ExecutorService waitingWorker) {
3133
this(waitingWorker, new DefaultControllerRateLimiter<>());
3234
}
3335

36+
/** Creates a queue using a caller-owned executor that is not shut down by this queue. */
3437
public DefaultRateLimitingQueue(ExecutorService waitingWorker, RateLimiter<T> rateLimiter) {
3538
super(waitingWorker);
3639
this.rateLimiter = rateLimiter;
Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
/*
2+
Copyright 2026 The Kubernetes Authors.
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
http://www.apache.org/licenses/LICENSE-2.0
7+
Unless required by applicable law or agreed to in writing, software
8+
distributed under the License is distributed on an "AS IS" BASIS,
9+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
See the License for the specific language governing permissions and
11+
limitations under the License.
12+
*/
13+
package io.kubernetes.client.extended.workqueue;
14+
15+
import static org.assertj.core.api.Assertions.assertThat;
16+
17+
import io.kubernetes.client.extended.controller.DefaultController;
18+
import io.kubernetes.client.extended.controller.builder.ControllerBuilder;
19+
import io.kubernetes.client.extended.controller.builder.DefaultControllerBuilder;
20+
import io.kubernetes.client.extended.controller.reconciler.Result;
21+
import io.kubernetes.client.extended.wait.Wait;
22+
import io.kubernetes.client.extended.workqueue.ratelimiter.DefaultControllerRateLimiter;
23+
import java.lang.reflect.Field;
24+
import java.time.Duration;
25+
import java.util.concurrent.CountDownLatch;
26+
import java.util.concurrent.ExecutorService;
27+
import java.util.concurrent.Executors;
28+
import java.util.concurrent.Future;
29+
import java.util.concurrent.TimeUnit;
30+
import org.junit.jupiter.api.Test;
31+
import org.junit.jupiter.params.ParameterizedTest;
32+
import org.junit.jupiter.params.provider.ValueSource;
33+
34+
class DefaultDelayingQueueShutdownTest {
35+
36+
@ParameterizedTest
37+
@ValueSource(booleans = {false, true})
38+
void defaultDelayingQueueTerminates(boolean delayed) throws Exception {
39+
assertOwnedExecutorTerminates(new DefaultDelayingQueue<>(), delayed);
40+
}
41+
42+
@ParameterizedTest
43+
@ValueSource(booleans = {false, true})
44+
void defaultRateLimitingQueueTerminates(boolean delayed) throws Exception {
45+
assertOwnedExecutorTerminates(new DefaultRateLimitingQueue<>(), delayed);
46+
}
47+
48+
private void assertOwnedExecutorTerminates(DefaultDelayingQueue<String> queue, boolean delayed)
49+
throws Exception {
50+
ExecutorService executor = ownedExecutor(queue);
51+
try {
52+
if (delayed) {
53+
queue.addAfter("later", Duration.ofDays(1));
54+
assertThat(
55+
Wait.poll(
56+
Duration.ofMillis(10),
57+
Duration.ofSeconds(5),
58+
() -> queue.waitingForAddQueue.isEmpty()))
59+
.isTrue();
60+
}
61+
queue.shutDown();
62+
queue.shutDown();
63+
assertThat(queue.isShuttingDown()).isTrue();
64+
assertThat(executor.isShutdown()).isTrue();
65+
assertThat(executor.awaitTermination(5, TimeUnit.SECONDS)).isTrue();
66+
queue.addAfter("ignored", Duration.ZERO);
67+
assertThat(queue.get()).isNull();
68+
} finally {
69+
queue.shutDown();
70+
executor.shutdownNow();
71+
assertThat(executor.awaitTermination(5, TimeUnit.SECONDS)).isTrue();
72+
}
73+
}
74+
75+
@ParameterizedTest
76+
@ValueSource(ints = {0, 1, 2})
77+
void suppliedExecutorRemainsUsable(int constructor) throws Exception {
78+
ExecutorService executor = Executors.newSingleThreadExecutor();
79+
DefaultDelayingQueue<String> queue = suppliedQueue(executor, constructor);
80+
try {
81+
queue.addAfter("later", Duration.ofDays(1));
82+
assertThat(
83+
Wait.poll(
84+
Duration.ofMillis(10),
85+
Duration.ofSeconds(5),
86+
() -> queue.waitingForAddQueue.isEmpty()))
87+
.isTrue();
88+
queue.shutDown();
89+
queue.shutDown();
90+
assertThat(executor.isShutdown()).isFalse();
91+
// On a single-thread executor this can complete only after the waiting loop exits.
92+
assertThat(executor.submit(() -> "still usable").get(5, TimeUnit.SECONDS))
93+
.isEqualTo("still usable");
94+
} finally {
95+
queue.shutDown();
96+
executor.shutdownNow();
97+
assertThat(executor.awaitTermination(5, TimeUnit.SECONDS)).isTrue();
98+
}
99+
}
100+
101+
@Test
102+
void shutdownBeforeWaitingTaskStartsDoesNotInterruptOtherWork() throws Exception {
103+
ExecutorService executor = Executors.newSingleThreadExecutor();
104+
CountDownLatch started = new CountDownLatch(1);
105+
CountDownLatch release = new CountDownLatch(1);
106+
Future<Boolean> unrelated =
107+
executor.submit(
108+
() -> {
109+
started.countDown();
110+
return release.await(5, TimeUnit.SECONDS);
111+
});
112+
DefaultDelayingQueue<String> queue = new DefaultDelayingQueue<>(executor);
113+
try {
114+
assertThat(started.await(5, TimeUnit.SECONDS)).isTrue();
115+
queue.shutDown();
116+
queue.shutDown();
117+
Field field = DefaultDelayingQueue.class.getDeclaredField("waitingTask");
118+
field.setAccessible(true);
119+
assertThat(((Future<?>) field.get(queue)).isCancelled()).isTrue();
120+
assertThat(unrelated.isDone()).isFalse();
121+
release.countDown();
122+
assertThat(unrelated.get(5, TimeUnit.SECONDS)).isTrue();
123+
assertThat(executor.submit(() -> "still usable").get(5, TimeUnit.SECONDS))
124+
.isEqualTo("still usable");
125+
} finally {
126+
release.countDown();
127+
queue.shutDown();
128+
executor.shutdownNow();
129+
assertThat(executor.awaitTermination(5, TimeUnit.SECONDS)).isTrue();
130+
}
131+
}
132+
133+
@Test
134+
void controllerShutdownTerminatesQueueExecutor() throws Exception {
135+
DefaultController controller =
136+
(DefaultController)
137+
ControllerBuilder.defaultBuilder(null)
138+
.withReconciler(request -> new Result(false))
139+
.build();
140+
ExecutorService executor =
141+
ownedExecutor((DefaultDelayingQueue<?>) controller.getWorkQueue());
142+
try {
143+
controller.shutdown();
144+
controller.shutdown();
145+
assertThat(executor.awaitTermination(5, TimeUnit.SECONDS)).isTrue();
146+
} finally {
147+
controller.shutdown();
148+
executor.shutdownNow();
149+
assertThat(executor.awaitTermination(5, TimeUnit.SECONDS)).isTrue();
150+
}
151+
}
152+
153+
@Test
154+
void replacingControllerQueueTerminatesOriginalExecutor() throws Exception {
155+
DefaultControllerBuilder builder = ControllerBuilder.defaultBuilder(null);
156+
Field field = DefaultControllerBuilder.class.getDeclaredField("workQueue");
157+
field.setAccessible(true);
158+
DefaultDelayingQueue<?> queue = (DefaultDelayingQueue<?>) field.get(builder);
159+
ExecutorService executor = ownedExecutor(queue);
160+
try {
161+
builder.withWorkQueue(null);
162+
assertThat(executor.awaitTermination(5, TimeUnit.SECONDS)).isTrue();
163+
} finally {
164+
queue.shutDown();
165+
executor.shutdownNow();
166+
assertThat(executor.awaitTermination(5, TimeUnit.SECONDS)).isTrue();
167+
}
168+
}
169+
170+
private static ExecutorService ownedExecutor(DefaultDelayingQueue<?> queue) throws Exception {
171+
Field field = DefaultDelayingQueue.class.getDeclaredField("ownedWaitingWorker");
172+
field.setAccessible(true);
173+
return (ExecutorService) field.get(queue);
174+
}
175+
176+
private static DefaultDelayingQueue<String> suppliedQueue(
177+
ExecutorService executor, int constructor) {
178+
switch (constructor) {
179+
case 0:
180+
return new DefaultDelayingQueue<>(executor);
181+
case 1:
182+
return new DefaultRateLimitingQueue<>(executor);
183+
default:
184+
return new DefaultRateLimitingQueue<>(executor, new DefaultControllerRateLimiter<>());
185+
}
186+
}
187+
}

‎extended/src/test/java/io/kubernetes/client/extended/workqueue/DefaultDelayingQueueTest.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,21 @@
1717
import io.kubernetes.client.extended.wait.Wait;
1818
import java.time.Duration;
1919
import java.time.Instant;
20+
import org.junit.jupiter.api.AfterEach;
2021
import org.junit.jupiter.api.Test;
2122

2223
class DefaultDelayingQueueTest {
2324

25+
private final DefaultDelayingQueue<String> queue = new DefaultDelayingQueue<>();
26+
27+
@AfterEach
28+
void tearDown() {
29+
queue.shutDown();
30+
}
31+
2432
@Test
2533
void simpleDelayingQueue() throws Exception {
2634
final Instant staticTime = Instant.now();
27-
DefaultDelayingQueue<String> queue = new DefaultDelayingQueue<>();
2835
// Hold time still
2936
queue.injectTimeSource(
3037
() -> {
@@ -52,7 +59,6 @@ void simpleDelayingQueue() throws Exception {
5259
@Test
5360
void deduping() throws Exception {
5461
final Instant staticTime = Instant.now();
55-
DefaultDelayingQueue<String> queue = new DefaultDelayingQueue<>();
5662
String item = "foo";
5763

5864
// Hold time still
@@ -111,7 +117,6 @@ void deduping() throws Exception {
111117
@Test
112118
void copyShifting() throws Exception {
113119
final Instant staticTime = Instant.now();
114-
DefaultDelayingQueue<String> queue = new DefaultDelayingQueue<>();
115120
queue.injectTimeSource(
116121
() -> {
117122
return staticTime.toEpochMilli();

‎extended/src/test/java/io/kubernetes/client/extended/workqueue/DefaultRateLimitQueueTest.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,23 @@
1616

1717
import io.kubernetes.client.extended.workqueue.ratelimiter.RateLimiter;
1818
import java.time.Duration;
19+
import java.util.concurrent.ExecutorService;
1920
import java.util.concurrent.Executors;
21+
import org.junit.jupiter.api.AfterEach;
2022
import org.junit.jupiter.api.Test;
2123

2224
class DefaultRateLimitQueueTest {
2325

26+
private final ExecutorService executor = Executors.newSingleThreadExecutor();
27+
private final DefaultRateLimitingQueue<String> rlq =
28+
new DefaultRateLimitingQueue<>(executor, new MockRateLimiter<>());
29+
30+
@AfterEach
31+
void tearDown() {
32+
rlq.shutDown();
33+
executor.shutdownNow();
34+
}
35+
2436
private static class MockRateLimiter<T> implements RateLimiter<T> {
2537

2638
private int count;
@@ -47,9 +59,6 @@ public int numRequeues(Object item) {
4759

4860
@Test
4961
void simpleRateLimitQueue() throws Exception {
50-
MockRateLimiter<String> mockRateLimiter = new MockRateLimiter<>();
51-
DefaultRateLimitingQueue<String> rlq =
52-
new DefaultRateLimitingQueue<>(Executors.newSingleThreadExecutor(), mockRateLimiter);
5362
long t1 = System.nanoTime();
5463
rlq.addRateLimited("foo");
5564
rlq.get();

0 commit comments

Comments
 (0)