From 148ef94779bebb01386ddfeea9875598b03cbff0 Mon Sep 17 00:00:00 2001 From: wind57 Date: Mon, 27 Jul 2026 12:42:15 +0300 Subject: [PATCH 01/24] fix-1828: started work Signed-off-by: wind57 --- ...urationWatcherConfigurationProperties.java | 10 ++++ .../ConfigurationWatcherHaProperties.java | 56 +++++++++++++++++++ ...onWatcherConfigurationPropertiesTests.java | 38 +++++++++++++ 3 files changed, 104 insertions(+) create mode 100644 spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherHaProperties.java diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationProperties.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationProperties.java index b261ea28d2..4b5065b58e 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationProperties.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationProperties.java @@ -87,6 +87,8 @@ public class ConfigurationWatcherConfigurationProperties { private Integer actuatorPort = -1; + private ConfigurationWatcherHaProperties ha = new ConfigurationWatcherHaProperties(); + public String getActuatorPath() { return actuatorPath; } @@ -134,6 +136,14 @@ public void setRefreshStrategy(RefreshStrategy refreshStrategy) { this.refreshStrategy = refreshStrategy; } + public ConfigurationWatcherHaProperties getHa() { + return ha; + } + + public void setHa(ConfigurationWatcherHaProperties ha) { + this.ha = ha; + } + public enum RefreshStrategy { /** diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherHaProperties.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherHaProperties.java new file mode 100644 index 0000000000..bcfc9f8d74 --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherHaProperties.java @@ -0,0 +1,56 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher; + +/** + * Properties for watcher HA state persisted in a Lease. + * + * @author wind57 + */ +public class ConfigurationWatcherHaProperties { + + private boolean enabled; + + private String leaseName = "configuration-watcher-ha"; + + private String leaseNamespace; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public String getLeaseName() { + return leaseName; + } + + public void setLeaseName(String leaseName) { + this.leaseName = leaseName; + } + + public String getLeaseNamespace() { + return leaseNamespace; + } + + public void setLeaseNamespace(String leaseNamespace) { + this.leaseNamespace = leaseNamespace; + } + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationPropertiesTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationPropertiesTests.java index c15e8c5e01..3d98146143 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationPropertiesTests.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationPropertiesTests.java @@ -18,6 +18,10 @@ import org.junit.jupiter.api.Test; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Configuration; + import static org.assertj.core.api.Assertions.assertThat; /** @@ -34,4 +38,38 @@ void setActuatorPath() { assertThat(properties.getActuatorPath()).isEqualTo("/foo/bar"); } + @Test + void testWithDefaults() { + new ApplicationContextRunner().withUserConfiguration(Config.class).run(context -> { + ConfigurationWatcherConfigurationProperties props = context + .getBean(ConfigurationWatcherConfigurationProperties.class); + assertThat(props).isNotNull(); + assertThat(props.getHa().isEnabled()).isFalse(); + assertThat(props.getHa().getLeaseName()).isEqualTo("configuration-watcher-ha"); + assertThat(props.getHa().getLeaseNamespace()).isNull(); + }); + } + + @Test + void testWithNonDefaults() { + new ApplicationContextRunner().withUserConfiguration(Config.class) + .withPropertyValues("spring.cloud.kubernetes.configuration.watcher.ha.enabled=true", + "spring.cloud.kubernetes.configuration.watcher.ha.lease-name=custom-lease", + "spring.cloud.kubernetes.configuration.watcher.ha.lease-namespace=watcher-namespace") + .run(context -> { + ConfigurationWatcherConfigurationProperties props = context + .getBean(ConfigurationWatcherConfigurationProperties.class); + assertThat(props).isNotNull(); + assertThat(props.getHa().isEnabled()).isTrue(); + assertThat(props.getHa().getLeaseName()).isEqualTo("custom-lease"); + assertThat(props.getHa().getLeaseNamespace()).isEqualTo("watcher-namespace"); + }); + } + + @Configuration + @EnableConfigurationProperties(ConfigurationWatcherConfigurationProperties.class) + static class Config { + + } + } From 250a69987df0d0a8897bbe57d1801aad2fddee31 Mon Sep 17 00:00:00 2001 From: wind57 Date: Fri, 31 Jul 2026 18:09:09 +0300 Subject: [PATCH 02/24] store implementation Signed-off-by: wind57 --- ...urationWatcherConfigurationProperties.java | 1 + .../ConfigurationWatcherHaProperties.java | 4 +- .../watcher/ha/ConfigurationWatcherState.java | 30 +++ .../ha/ConfigurationWatcherStateStore.java | 30 +++ .../LeaseConfigurationWatcherStateStore.java | 156 +++++++++++++++ ...onWatcherConfigurationPropertiesTests.java | 2 +- ...seConfigurationWatcherStateStoreTests.java | 179 ++++++++++++++++++ 7 files changed, 399 insertions(+), 3 deletions(-) rename spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/{ => ha}/ConfigurationWatcherHaProperties.java (96%) create mode 100644 spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherState.java create mode 100644 spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherStateStore.java create mode 100644 spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java create mode 100644 spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStoreTests.java diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationProperties.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationProperties.java index 4b5065b58e..c67cb8f025 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationProperties.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationProperties.java @@ -21,6 +21,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.convert.DurationUnit; +import org.springframework.cloud.kubernetes.configuration.watcher.ha.ConfigurationWatcherHaProperties; /** * @author Ryan Baxter diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherHaProperties.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHaProperties.java similarity index 96% rename from spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherHaProperties.java rename to spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHaProperties.java index bcfc9f8d74..fd888b6267 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherHaProperties.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHaProperties.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.cloud.kubernetes.configuration.watcher; +package org.springframework.cloud.kubernetes.configuration.watcher.ha; /** * Properties for watcher HA state persisted in a Lease. @@ -27,7 +27,7 @@ public class ConfigurationWatcherHaProperties { private String leaseName = "configuration-watcher-ha"; - private String leaseNamespace; + private String leaseNamespace = "default"; public boolean isEnabled() { return enabled; diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherState.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherState.java new file mode 100644 index 0000000000..749be2f1a2 --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherState.java @@ -0,0 +1,30 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher.ha; + +/** + * Persisted HA watcher state. + * + * @param configMapResourceVersion last processed configmap resource version + * @param secretResourceVersion last processed secret resource version + * @author wind57 + */ +record ConfigurationWatcherState(String configMapResourceVersion, String secretResourceVersion) { + + static final ConfigurationWatcherState EMPTY = new ConfigurationWatcherState(null, null); + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherStateStore.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherStateStore.java new file mode 100644 index 0000000000..3c49f4b79a --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherStateStore.java @@ -0,0 +1,30 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher.ha; + +/** + * Persistent storage for HA watcher state shared across watcher instances. + * + * @author wind57 + */ +sealed interface ConfigurationWatcherStateStore permits LeaseConfigurationWatcherStateStore { + + ConfigurationWatcherState readOrCreate(); + + void write(ConfigurationWatcherState state); + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java new file mode 100644 index 0000000000..4174b0e23b --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java @@ -0,0 +1,156 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher.ha; + +import java.util.HashMap; +import java.util.Map; + +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.CoordinationV1Api; +import io.kubernetes.client.openapi.models.V1Lease; +import io.kubernetes.client.openapi.models.V1ObjectMeta; + +import org.springframework.core.log.LogAccessor; +import org.springframework.util.StringUtils; + +/** + * Lease-backed persistent store for watcher HA state. + * + *

+ * All methods in this class are expected to be called only by the watcher instance that + * currently holds leadership. Follower instances must not read or write HA state through + * this store. + * + * @author wind57 + */ +final class LeaseConfigurationWatcherStateStore implements ConfigurationWatcherStateStore { + + private static final LogAccessor LOG = new LogAccessor(LeaseConfigurationWatcherStateStore.class); + + private static final String CONFIGMAP_ANNOTATION = "spring.cloud.kubernetes.configuration.watcher.configmap-resource-version"; + + private static final String SECRET_ANNOTATION = "spring.cloud.kubernetes.configuration.watcher.secret-resource-version"; + + private final CoordinationV1Api api; + + private final ConfigurationWatcherHaProperties properties; + + LeaseConfigurationWatcherStateStore(CoordinationV1Api api, ConfigurationWatcherHaProperties properties) { + this.api = api; + this.properties = properties; + } + + @Override + public ConfigurationWatcherState readOrCreate() { + + String leaseName = properties.getLeaseName(); + String leaseNamespace = namespace(); + LOG.debug("Reading lease with name: " + leaseName + " in namespace: " + leaseNamespace); + + try { + V1Lease lease = api.readNamespacedLease(leaseName, leaseNamespace).execute(); + Map annotations = existingAnnotations(lease); + return new ConfigurationWatcherState(annotations.get(CONFIGMAP_ANNOTATION), + annotations.get(SECRET_ANNOTATION)); + } + catch (ApiException e) { + if (e.getCode() == 404) { + createLease(leaseName, leaseNamespace); + return ConfigurationWatcherState.EMPTY; + } + throw new IllegalStateException("Failed to read watcher HA lease '" + properties.getLeaseName() + + "' in namespace '" + leaseNamespace + "'", e); + } + } + + @Override + public void write(ConfigurationWatcherState state) { + try { + + String leaseName = properties.getLeaseName(); + String leaseNamespace = namespace(); + LOG.debug("Updating lease with name: " + leaseName + " in namespace: " + leaseNamespace); + + V1Lease currentLease = api.readNamespacedLease(leaseName, leaseNamespace).execute(); + // add the annotations + V1Lease updatedLease = updatedLease(currentLease, state); + api.replaceNamespacedLease(leaseName, leaseNamespace, updatedLease).execute(); + } + catch (ApiException e) { + LOG.error(e, () -> "Failed to write to the lease because : " + e.getResponseBody()); + throw new IllegalStateException("Failed to write watcher HA lease '" + properties.getLeaseName() + + "' in namespace '" + namespace() + "'", e); + } + } + + private void createLease(String leaseName, String leaseNamespace) { + try { + LOG.info(() -> "Creating watcher HA lease with name : " + leaseName + " in namespace : " + leaseNamespace); + api.createNamespacedLease(leaseNamespace, newLease(leaseName, leaseNamespace)).execute(); + } + catch (ApiException e) { + LOG.error(e, () -> "Failed to create watcher HA lease '" + e.getResponseBody()); + throw new IllegalStateException("Failed to create watcher HA lease '" + properties.getLeaseName() + + "' in namespace '" + namespace() + "'", e); + } + } + + private V1Lease updatedLease(V1Lease lease, ConfigurationWatcherState state) { + V1ObjectMeta metadata = lease.getMetadata(); + Map currentLeaseAnnotations = existingAnnotations(lease); + metadata.setAnnotations(updateAnnotations(currentLeaseAnnotations, state)); + return lease; + } + + private V1Lease newLease(String leaseName, String leaseNamespace) { + V1ObjectMeta metadata = new V1ObjectMeta(); + metadata.setName(leaseName); + metadata.setNamespace(leaseNamespace); + return new V1Lease().metadata(metadata); + } + + private static Map existingAnnotations(V1Lease lease) { + if (lease.getMetadata() == null || lease.getMetadata().getAnnotations() == null) { + LOG.warn(() -> "Lease has no annotations"); + return Map.of(); + } + return lease.getMetadata().getAnnotations(); + } + + private static Map updateAnnotations(Map existingAnnotations, + ConfigurationWatcherState state) { + Map annotations = new HashMap<>(existingAnnotations); + + String configMapResourceVersion = state.configMapResourceVersion(); + if (configMapResourceVersion != null) { + annotations.put(CONFIGMAP_ANNOTATION, configMapResourceVersion); + } + + String secretResourceVersion = state.secretResourceVersion(); + if (secretResourceVersion != null) { + annotations.put(SECRET_ANNOTATION, secretResourceVersion); + } + + return annotations; + } + + private String namespace() { + String leaseNamespaceFromProperties = properties.getLeaseNamespace(); + return StringUtils.hasText(leaseNamespaceFromProperties) ? leaseNamespaceFromProperties : "default"; + } + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationPropertiesTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationPropertiesTests.java index 3d98146143..04485bee68 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationPropertiesTests.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherConfigurationPropertiesTests.java @@ -46,7 +46,7 @@ void testWithDefaults() { assertThat(props).isNotNull(); assertThat(props.getHa().isEnabled()).isFalse(); assertThat(props.getHa().getLeaseName()).isEqualTo("configuration-watcher-ha"); - assertThat(props.getHa().getLeaseNamespace()).isNull(); + assertThat(props.getHa().getLeaseNamespace()).isEqualTo("default"); }); } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStoreTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStoreTests.java new file mode 100644 index 0000000000..605ca84449 --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStoreTests.java @@ -0,0 +1,179 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher.ha; + +import java.util.HashMap; +import java.util.Map; + +import com.github.tomakehurst.wiremock.WireMockServer; +import io.kubernetes.client.openapi.apis.CoordinationV1Api; +import io.kubernetes.client.openapi.models.V1Lease; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import io.kubernetes.client.util.ClientBuilder; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.configureFor; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.matchingJsonPath; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.put; +import static com.github.tomakehurst.wiremock.client.WireMock.putRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; +import static io.kubernetes.client.openapi.JSON.serialize; +import static org.assertj.core.api.Assertions.assertThat; + +class LeaseConfigurationWatcherStateStoreTests { + + private static final String LEASE_NAME = "configuration-watcher-ha"; + + private static final String LEASE_NAMESPACE = "default"; + + private static final String LEASE_URL = "/apis/coordination.k8s.io/v1/namespaces/" + LEASE_NAMESPACE + "/leases/" + + LEASE_NAME; + + private static final String LEASE_COLLECTION_URL = "/apis/coordination.k8s.io/v1/namespaces/" + LEASE_NAMESPACE + + "/leases"; + + private static final String CONFIGMAP_RESOURCE_VERSION_ANNOTATION = "spring.cloud.kubernetes.configuration.watcher.configmap-resource-version"; + + private static final String SECRET_RESOURCE_VERSION_ANNOTATION = "spring.cloud.kubernetes.configuration.watcher.secret-resource-version"; + + private static WireMockServer wireMockServer; + + private static LeaseConfigurationWatcherStateStore stateStore; + + @BeforeAll + static void beforeAll() { + wireMockServer = new WireMockServer(options().dynamicPort()); + wireMockServer.start(); + configureFor("localhost", wireMockServer.port()); + stateStore = new LeaseConfigurationWatcherStateStore( + new CoordinationV1Api(new ClientBuilder().setBasePath(wireMockServer.baseUrl()).build()), + new ConfigurationWatcherHaProperties()); + } + + @AfterEach + void afterEach() { + wireMockServer.resetAll(); + } + + @AfterAll + static void afterAll() { + wireMockServer.stop(); + } + + /** + * 

+	 * 	- the HA lease already exists
+	 * 	- it stores the last seen ConfigMap and Secret resource versions
+	 * 	- readOrCreate returns those versions so the leader can resume from them
+	 *
+ */ + @Test + void readOrCreateReturnsExistingStateFromLeaseAnnotations() { + String configMapResourceVersion = "11"; + String secretResourceVersion = "22"; + + V1Lease lease = leaseWithAnnotations(configMapResourceVersion, secretResourceVersion); + stubFor(get(urlEqualTo(LEASE_URL)).willReturn(aResponse().withStatus(200).withBody(serialize(lease)))); + + ConfigurationWatcherState state = stateStore.readOrCreate(); + + assertThat(state.configMapResourceVersion()).isEqualTo(configMapResourceVersion); + assertThat(state.secretResourceVersion()).isEqualTo(secretResourceVersion); + wireMockServer.verify(getRequestedFor(urlEqualTo(LEASE_URL))); + } + + /** + * 
+	 * 	- the leader starts and no HA lease exists yet
+	 * 	- readOrCreate creates the lease in the configured namespace
+	 * 	- the returned state is empty because there is no previous checkpoint
+	 *
+ */ + @Test + void readOrCreateCreatesLeaseAndReturnsEmptyStateWhenLeaseIsMissing() { + V1Lease lease = new V1Lease().metadata(new V1ObjectMeta().name(LEASE_NAME).namespace(LEASE_NAMESPACE)); + + stubFor(get(urlEqualTo(LEASE_URL)).willReturn(aResponse().withStatus(404))); + stubFor(post(urlEqualTo(LEASE_COLLECTION_URL)) + .willReturn(aResponse().withStatus(200).withBody(serialize(lease)))); + + ConfigurationWatcherState state = stateStore.readOrCreate(); + + assertThat(state).isEqualTo(ConfigurationWatcherState.EMPTY); + wireMockServer.verify(getRequestedFor(urlEqualTo(LEASE_URL))); + wireMockServer.verify(postRequestedFor(urlEqualTo(LEASE_COLLECTION_URL)) + .withRequestBody(matchingJsonPath("$.metadata.name", equalTo(LEASE_NAME))) + .withRequestBody(matchingJsonPath("$.metadata.namespace", equalTo(LEASE_NAMESPACE)))); + } + + /** + * 
+	 * 	- the HA lease already contains a stored Secret resource version
+	 * 	- write stores the new ConfigMap resource version on the same lease
+	 * 	- existing annotations that are not overwritten stay in place
+	 *
+ */ + @Test + void writeUpdatesLeaseAnnotations() { + String existingSecretResourceVersion = "22"; + String configMapResourceVersion = "11"; + + V1Lease existingLease = leaseWithAnnotations(null, existingSecretResourceVersion); + V1Lease updatedLease = leaseWithAnnotations(configMapResourceVersion, existingSecretResourceVersion); + + stubFor(get(urlEqualTo(LEASE_URL)).willReturn(aResponse().withStatus(200).withBody(serialize(existingLease)))); + stubFor(put(urlEqualTo(LEASE_URL)).willReturn(aResponse().withStatus(200).withBody(serialize(updatedLease)))); + + stateStore.write(new ConfigurationWatcherState(configMapResourceVersion, null)); + + wireMockServer.verify(getRequestedFor(urlEqualTo(LEASE_URL))); + wireMockServer.verify(putRequestedFor(urlEqualTo(LEASE_URL)) + .withRequestBody( + matchingJsonPath("$.metadata.annotations." + jsonPath(CONFIGMAP_RESOURCE_VERSION_ANNOTATION), + equalTo(configMapResourceVersion))) + .withRequestBody(matchingJsonPath("$.metadata.annotations." + jsonPath(SECRET_RESOURCE_VERSION_ANNOTATION), + equalTo(existingSecretResourceVersion)))); + } + + private static V1Lease leaseWithAnnotations(String configMapResourceVersion, String secretResourceVersion) { + Map annotations = new HashMap<>(); + if (configMapResourceVersion != null) { + annotations.put(CONFIGMAP_RESOURCE_VERSION_ANNOTATION, configMapResourceVersion); + } + if (secretResourceVersion != null) { + annotations.put(SECRET_RESOURCE_VERSION_ANNOTATION, secretResourceVersion); + } + return new V1Lease() + .metadata(new V1ObjectMeta().name(LEASE_NAME).namespace(LEASE_NAMESPACE).annotations(annotations)); + } + + private static String jsonPath(String annotationName) { + return "['" + annotationName + "']"; + } + +} From 2caa8d2dcc6481c0734a5beb0b1cb53147933607 Mon Sep 17 00:00:00 2001 From: wind57 Date: Sat, 1 Aug 2026 16:28:03 +0300 Subject: [PATCH 03/24] more tests Signed-off-by: wind57 --- ...seConfigurationWatcherStateStoreTests.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStoreTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStoreTests.java index 605ca84449..7c8f5c8b83 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStoreTests.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStoreTests.java @@ -44,6 +44,7 @@ import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; import static io.kubernetes.client.openapi.JSON.serialize; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; class LeaseConfigurationWatcherStateStoreTests { @@ -160,6 +161,58 @@ void writeUpdatesLeaseAnnotations() { equalTo(existingSecretResourceVersion)))); } + /** + * 
+	 * 	- reading the HA lease fails with an error other than 404
+	 * 	- readOrCreate must fail instead of pretending that no previous state exists
+	 *
+ */ + @Test + void readOrCreateThrowsWhenLeaseReadFailsWithNon404() { + stubFor(get(urlEqualTo(LEASE_URL)).willReturn(aResponse().withStatus(500).withBody("boom"))); + + assertThatThrownBy(() -> stateStore.readOrCreate()).isInstanceOf(IllegalStateException.class) + .hasMessage("Failed to read watcher HA lease '" + LEASE_NAME + "' in namespace '" + LEASE_NAMESPACE + "'"); + } + + /** + * 
+	 * 	- the leader starts and the HA lease does not exist yet
+	 * 	- creating that lease fails
+	 * 	- readOrCreate must fail instead of returning an empty checkpoint
+	 *
+ */ + @Test + void readOrCreateThrowsWhenMissingLeaseCanNotBeCreated() { + stubFor(get(urlEqualTo(LEASE_URL)).willReturn(aResponse().withStatus(404))); + stubFor(post(urlEqualTo(LEASE_COLLECTION_URL)).willReturn(aResponse().withStatus(500).withBody("boom"))); + + assertThatThrownBy(() -> stateStore.readOrCreate()).isInstanceOf(IllegalStateException.class) + .hasMessage("Failed to create watcher HA lease '" + LEASE_NAME + "' in namespace '" + LEASE_NAMESPACE + "'"); + } + + /** + * 
+	 * 	- the existing HA lease is read successfully
+	 * 	- replacing it with the updated annotations fails
+	 * 	- write must surface that failure to the caller
+	 *
+ */ + @Test + void writeThrowsWhenLeaseUpdateFails() { + String existingSecretResourceVersion = "22"; + String configMapResourceVersion = "11"; + + V1Lease existingLease = leaseWithAnnotations(null, existingSecretResourceVersion); + + stubFor(get(urlEqualTo(LEASE_URL)).willReturn(aResponse().withStatus(200).withBody(serialize(existingLease)))); + stubFor(put(urlEqualTo(LEASE_URL)).willReturn(aResponse().withStatus(500).withBody("boom"))); + + assertThatThrownBy(() -> stateStore.write(new ConfigurationWatcherState(configMapResourceVersion, null))) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Failed to write watcher HA lease '" + LEASE_NAME + "' in namespace '" + LEASE_NAMESPACE + "'"); + } + private static V1Lease leaseWithAnnotations(String configMapResourceVersion, String secretResourceVersion) { Map annotations = new HashMap<>(); if (configMapResourceVersion != null) { From 5b2ed4c06e6b948c0a7f7a0210ea305bc437bf93 Mon Sep 17 00:00:00 2001 From: wind57 Date: Mon, 3 Aug 2026 15:09:21 +0300 Subject: [PATCH 04/24] next dev iteration: added ha support in the detectors Signed-off-by: wind57 --- ...ientEventBasedConfigMapChangeDetector.java | 95 +++++++++++++------ ...ClientEventBasedSecretsChangeDetector.java | 58 ++++++++--- ...ventBasedConfigMapChangeDetectorTests.java | 84 +++++++++++++++- ...tEventBasedSecretsChangeDetectorTests.java | 81 +++++++++++++++- .../ConfigMapWatcherChangeDetector.java | 3 +- .../watcher/SecretsWatcherChangeDetector.java | 3 +- 6 files changed, 273 insertions(+), 51 deletions(-) diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java index 69e90a452e..b9a051a5d7 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java @@ -33,7 +33,6 @@ import io.kubernetes.client.util.CallGeneratorParams; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; -import org.apache.commons.logging.LogFactory; import org.springframework.cloud.kubernetes.client.config.KubernetesClientConfigMapPropertySource; import org.springframework.cloud.kubernetes.client.config.KubernetesClientConfigMapPropertySourceLocator; @@ -54,8 +53,7 @@ */ public class KubernetesClientEventBasedConfigMapChangeDetector extends ConfigurationChangeDetector { - private static final LogAccessor LOG = new LogAccessor( - LogFactory.getLog(KubernetesClientEventBasedConfigMapChangeDetector.class)); + private static final LogAccessor LOG = new LogAccessor(KubernetesClientEventBasedConfigMapChangeDetector.class); private final CoreV1Api coreV1Api; @@ -77,6 +75,12 @@ public class KubernetesClientEventBasedConfigMapChangeDetector extends Configura private final Map configMapsLabels; + // HA enabled for configuration watcher + private final boolean haEnabled; + + // informers already running (skip starting more informers) + private volatile boolean running; + private final ResourceEventHandler handler = new ResourceEventHandler<>() { @Override @@ -110,6 +114,13 @@ public KubernetesClientEventBasedConfigMapChangeDetector(CoreV1Api coreV1Api, Co ConfigReloadProperties properties, ConfigurationUpdateStrategy strategy, KubernetesClientConfigMapPropertySourceLocator propertySourceLocator, KubernetesNamespaceProvider kubernetesNamespaceProvider) { + this(coreV1Api, environment, properties, strategy, propertySourceLocator, kubernetesNamespaceProvider, false); + } + + public KubernetesClientEventBasedConfigMapChangeDetector(CoreV1Api coreV1Api, ConfigurableEnvironment environment, + ConfigReloadProperties properties, ConfigurationUpdateStrategy strategy, + KubernetesClientConfigMapPropertySourceLocator propertySourceLocator, + KubernetesNamespaceProvider kubernetesNamespaceProvider, boolean haEnabled) { super(strategy); this.environment = environment; this.propertySourceLocator = propertySourceLocator; @@ -118,35 +129,47 @@ public KubernetesClientEventBasedConfigMapChangeDetector(CoreV1Api coreV1Api, Co this.enableReloadFiltering = properties.enableReloadFiltering(); this.monitoringConfigMaps = properties.monitoringConfigMaps(); this.configMapsLabels = properties.configMapsLabels(); + this.haEnabled = haEnabled; namespaces = namespaces(kubernetesNamespaceProvider, properties, "configmap"); } @PostConstruct void inform() { - if (monitoringConfigMaps) { - LOG.info(() -> "Kubernetes event-based configMap change detector activated"); - - Map labelSelector; - - if (enableReloadFiltering) { - LOG.warn(() -> "enable reload filtering is deprecated and will be removed in the next major release"); - LOG.warn(() -> "use spring.cloud.kubernetes.reload.config-maps-labels instead"); - if (!configMapsLabels.isEmpty()) { - LOG.warn(() -> "spring.cloud.kubernetes.reload.config-maps-labels is not empty, but " - + "spring.cloud.kubernetes.reload.enable-reload-filtering is enabled and will override the former"); - } - labelSelector = Map.of(ConfigReloadProperties.RELOAD_LABEL_FILTER, "true"); - } - else { - labelSelector = configMapsLabels; + // In HA mode, defer informer startup until this instance acquires leadership. + // The leader callback restores the persisted state and then starts the informers. + if (!haEnabled) { + start(); + } + } + + public final void start() { + if (running || !monitoringConfigMaps) { + return; + } + + LOG.info(() -> "Kubernetes event-based configMap change detector activated"); + + Map labelSelector; + + if (enableReloadFiltering) { + LOG.warn(() -> "enable reload filtering is deprecated and will be removed in the next major release"); + LOG.warn(() -> "use spring.cloud.kubernetes.reload.config-maps-labels instead"); + if (!configMapsLabels.isEmpty()) { + LOG.warn(() -> "spring.cloud.kubernetes.reload.config-maps-labels is not empty, but " + + "spring.cloud.kubernetes.reload.enable-reload-filtering is enabled and will override the former"); } + labelSelector = Map.of(ConfigReloadProperties.RELOAD_LABEL_FILTER, "true"); + } + else { + labelSelector = configMapsLabels; + } - namespaces.forEach(namespace -> { - SharedIndexInformer informer; + namespaces.forEach(namespace -> { + SharedIndexInformer informer; - SharedInformerFactory factory = new SharedInformerFactory(apiClient); - factories.add(factory); - informer = factory + SharedInformerFactory factory = new SharedInformerFactory(apiClient); + factories.add(factory); + informer = factory .sharedIndexInformerFor((CallGeneratorParams params) -> coreV1Api.listNamespacedConfigMap(namespace) .timeoutSeconds(params.timeoutSeconds) .resourceVersion(params.resourceVersion) @@ -154,21 +177,31 @@ void inform() { .labelSelector(labelSelector(labelSelector)) .buildCall(null), V1ConfigMap.class, V1ConfigMapList.class); - LOG.debug(() -> "added configmap informer for namespace : " + namespace + " with labels : " - + labelSelector); + LOG.debug(() -> "added configmap informer for namespace : " + namespace + " with labels : " + + labelSelector); - informer.addEventHandler(handler); - informers.add(informer); - factory.startAllRegisteredInformers(); - }); - } + informer.addEventHandler(handler); + informers.add(informer); + factory.startAllRegisteredInformers(); + }); + running = true; } @PreDestroy void shutdown() { + stop(); + } + + public final void stop() { + if (!running) { + return; + } informers.forEach(SharedIndexInformer::stop); factories.forEach(SharedInformerFactory::stopAllRegisteredInformers); + informers.clear(); + factories.clear(); + running = false; } protected void onEvent(KubernetesObject configMap) { diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java index 63337cad08..7cab05764b 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java @@ -78,6 +78,12 @@ public class KubernetesClientEventBasedSecretsChangeDetector extends Configurati private final Map secretsLabels; + // HA enabled for configuration watcher + private final boolean haEnabled; + + // informers already running (skip starting more informers) + private volatile boolean running; + private final ResourceEventHandler handler = new ResourceEventHandler<>() { @Override @@ -112,6 +118,13 @@ public KubernetesClientEventBasedSecretsChangeDetector(CoreV1Api coreV1Api, Conf ConfigReloadProperties properties, ConfigurationUpdateStrategy strategy, KubernetesClientSecretsPropertySourceLocator propertySourceLocator, KubernetesNamespaceProvider kubernetesNamespaceProvider) { + this(coreV1Api, environment, properties, strategy, propertySourceLocator, kubernetesNamespaceProvider, false); + } + + public KubernetesClientEventBasedSecretsChangeDetector(CoreV1Api coreV1Api, ConfigurableEnvironment environment, + ConfigReloadProperties properties, ConfigurationUpdateStrategy strategy, + KubernetesClientSecretsPropertySourceLocator propertySourceLocator, + KubernetesNamespaceProvider kubernetesNamespaceProvider, boolean haEnabled) { super(strategy); this.environment = environment; this.propertySourceLocator = propertySourceLocator; @@ -120,11 +133,23 @@ public KubernetesClientEventBasedSecretsChangeDetector(CoreV1Api coreV1Api, Conf this.enableReloadFiltering = properties.enableReloadFiltering(); this.monitoringSecrets = properties.monitoringSecrets(); this.secretsLabels = properties.secretsLabels(); + this.haEnabled = haEnabled; namespaces = namespaces(kubernetesNamespaceProvider, properties, "secret"); } @PostConstruct void inform() { + // In HA mode, defer informer startup until this instance acquires leadership. + // The leader callback restores the persisted state and then starts the informers. + if (!haEnabled) { + start(); + } + } + + public final void start() { + if (running || !monitoringSecrets) { + return; + } LOG.info(() -> "Kubernetes event-based secrets change detector activated"); Map labelSelector; @@ -142,13 +167,12 @@ void inform() { labelSelector = secretsLabels; } - if (monitoringSecrets) { - namespaces.forEach(namespace -> { - SharedIndexInformer informer; + namespaces.forEach(namespace -> { + SharedIndexInformer informer; - SharedInformerFactory factory = new SharedInformerFactory(apiClient); - factories.add(factory); - informer = factory + SharedInformerFactory factory = new SharedInformerFactory(apiClient); + factories.add(factory); + informer = factory .sharedIndexInformerFor((CallGeneratorParams params) -> coreV1Api.listNamespacedSecret(namespace) .timeoutSeconds(params.timeoutSeconds) .resourceVersion(params.resourceVersion) @@ -156,20 +180,30 @@ void inform() { .labelSelector(labelSelector(labelSelector)) .buildCall(null), V1Secret.class, V1SecretList.class); - LOG.debug(() -> "secret informer for namespace : " + namespace + " with filter : " + secretsLabels); + LOG.debug(() -> "secret informer for namespace : " + namespace + " with filter : " + secretsLabels); - informer.addEventHandler(handler); - informers.add(informer); - factory.startAllRegisteredInformers(); - }); - } + informer.addEventHandler(handler); + informers.add(informer); + factory.startAllRegisteredInformers(); + }); + running = true; } @PreDestroy void shutdown() { + stop(); + } + + public final void stop() { + if (!running) { + return; + } informers.forEach(SharedIndexInformer::stop); factories.forEach(SharedInformerFactory::stopAllRegisteredInformers); + informers.clear(); + factories.clear(); + running = false; } protected void onEvent(KubernetesObject secret) { diff --git a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java index b96acf1697..3d7e5b5a8d 100644 --- a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java +++ b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java @@ -52,6 +52,7 @@ import static io.kubernetes.client.informer.EventType.DELETED; import static io.kubernetes.client.informer.EventType.MODIFIED; import static io.kubernetes.client.util.Watch.Response; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.springframework.cloud.kubernetes.commons.config.Constants.APPLICATION_PROPERTIES; @@ -143,13 +144,82 @@ void watch() { .withQueryParam("resourceVersion", equalTo("3")) .willReturn(aResponse().withStatus(200).withBody(JSON.serialize(watchResponseThree)))); - // ------------------------------------------------------------------------------------------------------------ // 4. assertions + changeDetectorAssert(false); + } + + /** + * 
+	 *     - HA mode is enabled, so inform() does not start the informer.
+	 *     - an explicit start() starts the informer and it processes the events.
+	 *
+ */ + @Test + void watchStartsOnlyAfterExplicitStartInHaMode() { + + // ------------------------------------------------------------------------------------------------------------ + // 0. initial request of the informer ( resourceVersion=0 ) + Map myConfigInitial = Map.of(APPLICATION_PROPERTIES, + "spring.cloud.kubernetes.configuration.watcher.refreshDelay=0"); + + V1ConfigMap myConfigMapInitial = new V1ConfigMap() + .metadata(new V1ObjectMeta().namespace("default").name("my-configmap")) + .data(myConfigInitial); + V1ConfigMapList myConfigMapListInitial = new V1ConfigMapList().metadata(new V1ListMeta().resourceVersion("1")) + .items(List.of(myConfigMapInitial)); + + stubFor(get(urlMatching("^/api/v1/namespaces/default/configmaps.*")).withQueryParam("watch", equalTo("false")) + .withQueryParam("resourceVersion", equalTo("0")) + .willReturn(aResponse().withStatus(200).withBody(JSON.serialize(myConfigMapListInitial)))); + + // ------------------------------------------------------------------------------------------------------------ + // 1. first watch response to request with resourceVersion=1 + Map myConfigChanged = Map.of(APPLICATION_PROPERTIES, + "spring.cloud.kubernetes.configuration.watcher.refreshDelay=1"); + + V1ConfigMap myConfigMapChanged = new V1ConfigMap() + .metadata(new V1ObjectMeta().namespace("default").name("my-configmap").resourceVersion("2")) + .data(myConfigChanged); + + Response watchResponseOne = new Response<>(MODIFIED.name(), myConfigMapChanged); + + stubFor(get(urlMatching("^/api/v1/namespaces/default/configmaps.*")).withQueryParam("watch", equalTo("true")) + .withQueryParam("resourceVersion", equalTo("1")) + .willReturn(aResponse().withStatus(200).withBody(JSON.serialize(watchResponseOne)))); + + // ------------------------------------------------------------------------------------------------------------ + // 2. second watch response to request with resourceVersion=2 + Map newConfigAdded = Map.of(APPLICATION_PROPERTIES, "debug=true"); + + V1ConfigMap newConfigMapAdded = new V1ConfigMap() + .metadata(new V1ObjectMeta().namespace("default").name("new-configmap").resourceVersion("3")) + .data(newConfigAdded); + + Response watchResponseTwo = new Response<>(ADDED.name(), newConfigMapAdded); + + stubFor(get(urlMatching("^/api/v1/namespaces/default/configmaps.*")).withQueryParam("watch", equalTo("true")) + .withQueryParam("resourceVersion", equalTo("2")) + .willReturn(aResponse().withStatus(200).withBody(JSON.serialize(watchResponseTwo)))); - changeDetectorAssert(); + // ------------------------------------------------------------------------------------------------------------ + // 3. third watch response to request with resourceVersion=3 + Map newConfigDeleted = Map.of(APPLICATION_PROPERTIES, "debug=true"); + + V1ConfigMap newConfigMapDeleted = new V1ConfigMap() + .metadata(new V1ObjectMeta().namespace("default").name("new-configmap").resourceVersion("4")) + .data(newConfigDeleted); + + Response watchResponseThree = new Response<>(DELETED.name(), newConfigMapDeleted); + + stubFor(get(urlMatching("^/api/v1/namespaces/default/configmaps.*")).withQueryParam("watch", equalTo("true")) + .withQueryParam("resourceVersion", equalTo("3")) + .willReturn(aResponse().withStatus(200).withBody(JSON.serialize(watchResponseThree)))); + + // 4. assertions + changeDetectorAssert(true); } - private void changeDetectorAssert() { + private void changeDetectorAssert(boolean haEnabled) { // coreV1Api ApiClient apiClient = new ClientBuilder().setBasePath("http://localhost:" + wireMockServer.port()).build(); @@ -175,10 +245,16 @@ private void changeDetectorAssert() { // change detector KubernetesClientEventBasedConfigMapChangeDetector changeDetector = new KubernetesClientEventBasedConfigMapChangeDetector( - coreV1Api, environment, ConfigReloadProperties.DEFAULT, strategy, locator, kubernetesNamespaceProvider); + coreV1Api, environment, ConfigReloadProperties.DEFAULT, strategy, locator, kubernetesNamespaceProvider, + haEnabled); changeDetector.inform(); + if (haEnabled) { + assertThat(onEventCalls[0]).isZero(); + changeDetector.start(); + } + // all 4 events are caught Awaitilities.awaitUntil(10, 1000, () -> onEventCalls[0] == 4); diff --git a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java index e8bbd8fa44..381897f2f2 100644 --- a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java +++ b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java @@ -146,7 +146,75 @@ void watch() { // ------------------------------------------------------------------------------------------------------------ // 4. assertions - changeDetectorAssert(); + changeDetectorAssert(false); + } + + /** + * 
+	 *     - HA mode is enabled, so inform() does not start the informer.
+	 *     - an explicit start() starts the informer and it processes the events.
+	 *
+ */ + @Test + void watchStartsOnlyAfterExplicitStartInHaMode() { + + // ------------------------------------------------------------------------------------------------------------ + // 0. initial request of the informer ( resourceVersion=0 ) + + V1Secret dbPassword = new V1Secret().metadata(new V1ObjectMeta().name("db-password").resourceVersion("1")) + .putStringDataItem("password", Base64.getEncoder().encodeToString("p455w0rd".getBytes())) + .putDataItem("password", Base64.getEncoder().encode("p455w0rd".getBytes())) + .putStringDataItem("username", Base64.getEncoder().encodeToString("user".getBytes())) + .putDataItem("username", Base64.getEncoder().encode("user".getBytes())); + + V1SecretList secretList = new V1SecretList().metadata(new V1ListMeta().resourceVersion("1")) + .items(List.of(dbPassword)); + + stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).withQueryParam("watch", equalTo("false")) + .withQueryParam("resourceVersion", equalTo("0")) + .willReturn(aResponse().withStatus(200).withBody(JSON.serialize(secretList)))); + + // ------------------------------------------------------------------------------------------------------------ + // 1. first watch response to request with resourceVersion=1 + + V1Secret dbPasswordUpdated = new V1Secret() + .metadata(new V1ObjectMeta().name("db-password").resourceVersion("2")) + .putStringDataItem("password", Base64.getEncoder().encodeToString("p455w0rd2".getBytes())) + .putDataItem("password", Base64.getEncoder().encode("p455w0rd2".getBytes())) + .putStringDataItem("username", Base64.getEncoder().encodeToString("user".getBytes())) + .putDataItem("username", Base64.getEncoder().encode("user".getBytes())); + + Response watchResponse = new Response<>(MODIFIED.name(), dbPasswordUpdated); + + stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).withQueryParam("watch", equalTo("true")) + .withQueryParam("resourceVersion", equalTo("1")) + .willReturn(aResponse().withStatus(200).withBody(JSON.serialize(watchResponse)))); + + // ------------------------------------------------------------------------------------------------------------ + // 2. second watch response to request with resourceVersion=2 + + V1Secret rabbitPasswordAdded = new V1Secret().metadata(new V1ObjectMeta().name("rabbit-password")) + .putDataItem("rabbit-pw", Base64.getEncoder().encode("password".getBytes())); + + Response rabbitPasswordAddedResponse = new Response<>(ADDED.name(), rabbitPasswordAdded); + + stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).withQueryParam("watch", equalTo("true")) + .withQueryParam("resourceVersion", equalTo("2")) + .willReturn(aResponse().withStatus(200).withBody(JSON.serialize(rabbitPasswordAddedResponse)))); + + // ------------------------------------------------------------------------------------------------------------ + // 3. third watch response to request with resourceVersion=3 + + V1Secret rabbitPasswordDeleted = new V1Secret().metadata(new V1ObjectMeta().name("rabbit-password")) + .putDataItem("rabbit-pw", Base64.getEncoder().encode("password".getBytes())); + + Response rabbitPasswordDeletedResponse = new Response<>(DELETED.name(), rabbitPasswordDeleted); + + stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).withQueryParam("watch", equalTo("true")) + .withQueryParam("resourceVersion", equalTo("3")) + .willReturn(aResponse().withStatus(200).withBody(JSON.serialize(rabbitPasswordDeletedResponse)))); + + changeDetectorAssert(true); } /** @@ -260,6 +328,10 @@ void equalsEight() { } private void changeDetectorAssert() { + changeDetectorAssert(true); + } + + private void changeDetectorAssert(boolean haEnabled) { // coreV1Api ApiClient apiClient = new ClientBuilder().setBasePath("http://localhost:" + wireMockServer.port()).build(); @@ -290,10 +362,15 @@ private void changeDetectorAssert() { // change detector KubernetesClientEventBasedSecretsChangeDetector changeDetector = new KubernetesClientEventBasedSecretsChangeDetector( - coreV1Api, environment, properties, strategy, locator, kubernetesNamespaceProvider); + coreV1Api, environment, properties, strategy, locator, kubernetesNamespaceProvider, haEnabled); changeDetector.inform(); + if (haEnabled) { + Assertions.assertThat(onEventCalls[0]).isZero(); + changeDetector.start(); + } + // all 4 events are caught Awaitilities.awaitUntil(10, 1000, () -> onEventCalls[0] >= 4); diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigMapWatcherChangeDetector.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigMapWatcherChangeDetector.java index 5e0176d36d..824ae67559 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigMapWatcherChangeDetector.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigMapWatcherChangeDetector.java @@ -64,7 +64,8 @@ abstract sealed class ConfigMapWatcherChangeDetector extends KubernetesClientEve KubernetesNamespaceProvider kubernetesNamespaceProvider, ConfigurationWatcherConfigurationProperties k8SConfigurationProperties, ThreadPoolTaskExecutor threadPoolTaskExecutor) { - super(coreV1Api, environment, properties, strategy, propertySourceLocator, kubernetesNamespaceProvider); + super(coreV1Api, environment, properties, strategy, propertySourceLocator, kubernetesNamespaceProvider, + k8SConfigurationProperties.getHa().isEnabled()); scheduler = Schedulers.fromExecutor( newScheduledThreadPool(k8SConfigurationProperties.getThreadPoolSize(), threadPoolTaskExecutor)); this.k8SConfigurationProperties = k8SConfigurationProperties; diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/SecretsWatcherChangeDetector.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/SecretsWatcherChangeDetector.java index fa4bcad950..e8e517d0ae 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/SecretsWatcherChangeDetector.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/SecretsWatcherChangeDetector.java @@ -63,7 +63,8 @@ abstract sealed class SecretsWatcherChangeDetector extends KubernetesClientEvent KubernetesNamespaceProvider kubernetesNamespaceProvider, ConfigurationWatcherConfigurationProperties k8SConfigurationProperties, ThreadPoolTaskExecutor threadPoolTaskExecutor) { - super(coreV1Api, environment, properties, strategy, propertySourceLocator, kubernetesNamespaceProvider); + super(coreV1Api, environment, properties, strategy, propertySourceLocator, kubernetesNamespaceProvider, + k8SConfigurationProperties.getHa().isEnabled()); scheduler = Schedulers.fromExecutor( newScheduledThreadPool(k8SConfigurationProperties.getThreadPoolSize(), threadPoolTaskExecutor)); this.k8SConfigurationProperties = k8SConfigurationProperties; From 496a26577488fe826c937e98d90afd85999ed1d5 Mon Sep 17 00:00:00 2001 From: wind57 Date: Wed, 5 Aug 2026 13:14:28 +0300 Subject: [PATCH 05/24] added ha coordinator Signed-off-by: wind57 --- ...ConfigurationWatcherAutoConfiguration.java | 14 +++ ...tionalOnConfigurationWatcherHAEnabled.java | 41 +++++++ .../ha/ConfigurationWatcherHACoordinator.java | 69 ++++++++++++ ...onfigurationWatcherHACoordinatorTests.java | 105 ++++++++++++++++++ 4 files changed, 229 insertions(+) create mode 100644 spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConditionalOnConfigurationWatcherHAEnabled.java create mode 100644 spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java create mode 100644 spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherAutoConfiguration.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherAutoConfiguration.java index dbcfb5355c..d81d613620 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherAutoConfiguration.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherAutoConfiguration.java @@ -18,6 +18,7 @@ import io.kubernetes.client.openapi.apis.CoreV1Api; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.boot.autoconfigure.AutoConfigureAfter; import org.springframework.boot.autoconfigure.AutoConfigureBefore; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; @@ -28,9 +29,13 @@ import org.springframework.cloud.bus.BusStreamAutoConfiguration; import org.springframework.cloud.kubernetes.client.config.KubernetesClientConfigMapPropertySourceLocator; import org.springframework.cloud.kubernetes.client.config.KubernetesClientSecretsPropertySourceLocator; +import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedConfigMapChangeDetector; +import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedSecretsChangeDetector; import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider; import org.springframework.cloud.kubernetes.commons.config.reload.ConfigReloadProperties; import org.springframework.cloud.kubernetes.commons.config.reload.ConfigurationUpdateStrategy; +import org.springframework.cloud.kubernetes.configuration.watcher.ha.ConditionalOnConfigurationWatcherHAEnabled; +import org.springframework.cloud.kubernetes.configuration.watcher.ha.ConfigurationWatcherHACoordinator; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.AbstractEnvironment; @@ -55,6 +60,15 @@ WebClient webClient(WebClient.Builder webClientBuilder) { return webClientBuilder.build(); } + @Bean + @ConditionalOnMissingBean + @ConditionalOnConfigurationWatcherHAEnabled + ConfigurationWatcherHACoordinator configurationWatcherHACoordinator( + ObjectProvider configMapDetector, + ObjectProvider secretsDetector) { + return new ConfigurationWatcherHACoordinator(configMapDetector, secretsDetector); + } + @Bean @ConditionalOnMissingBean @ConditionalOnBean(KubernetesClientConfigMapPropertySourceLocator.class) diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConditionalOnConfigurationWatcherHAEnabled.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConditionalOnConfigurationWatcherHAEnabled.java new file mode 100644 index 0000000000..8cedcfa9be --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConditionalOnConfigurationWatcherHAEnabled.java @@ -0,0 +1,41 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher.ha; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +/** + * Provides a succinct conditional for enabling configuration watcher HA support. + * + * @author wind57 + */ +@Target({ ElementType.TYPE, ElementType.METHOD }) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Inherited +@ConditionalOnProperty(prefix = "spring.cloud.kubernetes.configuration.watcher.ha", name = "enabled", + havingValue = "true") +public @interface ConditionalOnConfigurationWatcherHAEnabled { + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java new file mode 100644 index 0000000000..744adcf5c0 --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java @@ -0,0 +1,69 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher.ha; + +import java.time.Instant; + +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedConfigMapChangeDetector; +import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedSecretsChangeDetector; +import org.springframework.cloud.kubernetes.commons.leader.election.events.StartLeadingEvent; +import org.springframework.cloud.kubernetes.commons.leader.election.events.StopLeadingEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.log.LogAccessor; + +/** + * Coordinates the lifecycle of the ConfigMap and Secret watchers. + * + * @author wind57 + */ +public final class ConfigurationWatcherHACoordinator { + + private static final LogAccessor LOG = new LogAccessor(ConfigurationWatcherHACoordinator.class); + + private final ObjectProvider configMapDetector; + + private final ObjectProvider secretsDetector; + + public ConfigurationWatcherHACoordinator( + ObjectProvider configMapDetector, + ObjectProvider secretsDetector) { + if (configMapDetector.getIfAvailable() == null && secretsDetector.getIfAvailable() == null) { + throw new IllegalStateException("Configuration watcher HA is enabled, but neither ConfigMap nor Secret " + + "watching is enabled"); + } + this.configMapDetector = configMapDetector; + this.secretsDetector = secretsDetector; + } + + @EventListener + void onStartLeading(StartLeadingEvent event) { + LOG.info(() -> "configuration watcher with identity : " + event.candidateIdentity() + + " became leader at : " + Instant.ofEpochMilli(event.getTimestamp())); + configMapDetector.ifAvailable(KubernetesClientEventBasedConfigMapChangeDetector::start); + secretsDetector.ifAvailable(KubernetesClientEventBasedSecretsChangeDetector::start); + } + + @EventListener + void onStopLeading(StopLeadingEvent event) { + LOG.info(() -> "configuration watcher with identity : " + event.candidateIdentity() + + " stopped being a leader at : " + Instant.ofEpochMilli(event.getTimestamp())); + secretsDetector.ifAvailable(KubernetesClientEventBasedSecretsChangeDetector::stop); + configMapDetector.ifAvailable(KubernetesClientEventBasedConfigMapChangeDetector::stop); + } + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java new file mode 100644 index 0000000000..ae9131918c --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java @@ -0,0 +1,105 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher.ha; + +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.support.DefaultListableBeanFactory; +import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedConfigMapChangeDetector; +import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedSecretsChangeDetector; +import org.springframework.cloud.kubernetes.commons.leader.election.events.StartLeadingEvent; +import org.springframework.cloud.kubernetes.commons.leader.election.events.StopLeadingEvent; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +/** + * @author wind57 + */ +class ConfigurationWatcherHACoordinatorTests { + + @Test + void onStartLeadingStartsBothDetectors() { + KubernetesClientEventBasedConfigMapChangeDetector configMapDetector = mock( + KubernetesClientEventBasedConfigMapChangeDetector.class); + + KubernetesClientEventBasedSecretsChangeDetector secretsDetector = mock( + KubernetesClientEventBasedSecretsChangeDetector.class); + + ObjectProvider configMapProvider = provider( + KubernetesClientEventBasedConfigMapChangeDetector.class, configMapDetector); + + ObjectProvider secretsProvider = provider( + KubernetesClientEventBasedSecretsChangeDetector.class, secretsDetector); + + ConfigurationWatcherHACoordinator coordinator = new ConfigurationWatcherHACoordinator(configMapProvider, + secretsProvider); + + coordinator.onStartLeading(new StartLeadingEvent("candidate")); + + verify(configMapDetector).start(); + verify(secretsDetector).start(); + } + + @Test + void onStopLeadingStopsBothDetectors() { + KubernetesClientEventBasedConfigMapChangeDetector configMapDetector = mock( + KubernetesClientEventBasedConfigMapChangeDetector.class); + + KubernetesClientEventBasedSecretsChangeDetector secretsDetector = mock( + KubernetesClientEventBasedSecretsChangeDetector.class); + + ObjectProvider configMapProvider = provider( + KubernetesClientEventBasedConfigMapChangeDetector.class, configMapDetector); + + ObjectProvider secretsProvider = provider( + KubernetesClientEventBasedSecretsChangeDetector.class, secretsDetector); + + ConfigurationWatcherHACoordinator coordinator = new ConfigurationWatcherHACoordinator(configMapProvider, + secretsProvider); + + coordinator.onStopLeading(new StopLeadingEvent("candidate")); + + verify(configMapDetector).stop(); + verify(secretsDetector).stop(); + } + + @Test + void failsWhenNeitherDetectorIsAvailable() { + ObjectProvider configMapProvider = emptyProvider( + KubernetesClientEventBasedConfigMapChangeDetector.class); + ObjectProvider secretsProvider = emptyProvider( + KubernetesClientEventBasedSecretsChangeDetector.class); + + assertThatThrownBy(() -> new ConfigurationWatcherHACoordinator(configMapProvider, secretsProvider)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Configuration watcher HA is enabled, but neither ConfigMap nor Secret watching is enabled"); + } + + private static ObjectProvider provider(Class type, T value) { + DefaultListableBeanFactory beanFactory = new DefaultListableBeanFactory(); + beanFactory.registerSingleton("detector", value); + return beanFactory.getBeanProvider(type); + } + + private static ObjectProvider emptyProvider(Class type) { + return new DefaultListableBeanFactory().getBeanProvider(type); + } + +} From 8346a414d48574fe41b8f27e9ee245d4393d228b Mon Sep 17 00:00:00 2001 From: wind57 Date: Wed, 5 Aug 2026 13:38:24 +0300 Subject: [PATCH 06/24] added configuration Signed-off-by: wind57 --- ...ConfigurationWatcherAutoConfiguration.java | 14 ----- ...nfigurationWatcherHAAutoConfiguration.java | 62 +++++++++++++++++++ ...ot.autoconfigure.AutoConfiguration.imports | 1 + 3 files changed, 63 insertions(+), 14 deletions(-) create mode 100644 spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfiguration.java diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherAutoConfiguration.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherAutoConfiguration.java index d81d613620..dbcfb5355c 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherAutoConfiguration.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherAutoConfiguration.java @@ -18,7 +18,6 @@ import io.kubernetes.client.openapi.apis.CoreV1Api; -import org.springframework.beans.factory.ObjectProvider; import org.springframework.boot.autoconfigure.AutoConfigureAfter; import org.springframework.boot.autoconfigure.AutoConfigureBefore; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; @@ -29,13 +28,9 @@ import org.springframework.cloud.bus.BusStreamAutoConfiguration; import org.springframework.cloud.kubernetes.client.config.KubernetesClientConfigMapPropertySourceLocator; import org.springframework.cloud.kubernetes.client.config.KubernetesClientSecretsPropertySourceLocator; -import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedConfigMapChangeDetector; -import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedSecretsChangeDetector; import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider; import org.springframework.cloud.kubernetes.commons.config.reload.ConfigReloadProperties; import org.springframework.cloud.kubernetes.commons.config.reload.ConfigurationUpdateStrategy; -import org.springframework.cloud.kubernetes.configuration.watcher.ha.ConditionalOnConfigurationWatcherHAEnabled; -import org.springframework.cloud.kubernetes.configuration.watcher.ha.ConfigurationWatcherHACoordinator; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.AbstractEnvironment; @@ -60,15 +55,6 @@ WebClient webClient(WebClient.Builder webClientBuilder) { return webClientBuilder.build(); } - @Bean - @ConditionalOnMissingBean - @ConditionalOnConfigurationWatcherHAEnabled - ConfigurationWatcherHACoordinator configurationWatcherHACoordinator( - ObjectProvider configMapDetector, - ObjectProvider secretsDetector) { - return new ConfigurationWatcherHACoordinator(configMapDetector, secretsDetector); - } - @Bean @ConditionalOnMissingBean @ConditionalOnBean(KubernetesClientConfigMapPropertySourceLocator.class) diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfiguration.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfiguration.java new file mode 100644 index 0000000000..08a1a1fb4b --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfiguration.java @@ -0,0 +1,62 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher.ha; + +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.apis.CoordinationV1Api; + +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnCloudPlatform; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.cloud.CloudPlatform; +import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedConfigMapChangeDetector; +import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedSecretsChangeDetector; +import org.springframework.cloud.kubernetes.configuration.watcher.ConfigurationWatcherConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Configures the components required for configuration watcher HA. + * + *

Creates the persistent state store and the coordinator that starts and stops + * the watchers when leadership changes. + * + * @author wind57 + */ +@Configuration(proxyBeanMethods = false) +@ConditionalOnCloudPlatform(CloudPlatform.KUBERNETES) +@ConditionalOnConfigurationWatcherHAEnabled +@ConditionalOnBean(ApiClient.class) +class ConfigurationWatcherHAAutoConfiguration { + + @Bean + @ConditionalOnMissingBean + ConfigurationWatcherStateStore configurationWatcherStateStore(ApiClient apiClient, + ConfigurationWatcherConfigurationProperties properties) { + return new LeaseConfigurationWatcherStateStore(new CoordinationV1Api(apiClient), properties.getHa()); + } + + @Bean + @ConditionalOnMissingBean + ConfigurationWatcherHACoordinator configurationWatcherHACoordinator( + ObjectProvider configMapDetector, + ObjectProvider secretsDetector) { + return new ConfigurationWatcherHACoordinator(configMapDetector, secretsDetector); + } + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index 754b9a11f8..2e9281df24 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -3,3 +3,4 @@ org.springframework.cloud.kubernetes.configuration.watcher.ConfigUpdateStrategyA org.springframework.cloud.kubernetes.configuration.watcher.BusKafkaAutoConfiguration org.springframework.cloud.kubernetes.configuration.watcher.BusRabbitAutoConfiguration org.springframework.cloud.kubernetes.configuration.watcher.RefreshTriggerAutoConfiguration +org.springframework.cloud.kubernetes.configuration.watcher.ha.ConfigurationWatcherHAAutoConfiguration From 1032b8cc39d9aa053e3cdf6014825b0e3dd245fb Mon Sep 17 00:00:00 2001 From: wind57 Date: Mon, 17 Aug 2026 14:43:15 +0300 Subject: [PATCH 07/24] added the read path Signed-off-by: wind57 --- .../InformerResourceVersionResolver.java | 74 +++++++++++++++ ...ientEventBasedConfigMapChangeDetector.java | 25 ++--- ...ClientEventBasedSecretsChangeDetector.java | 19 ++-- .../InformerResourceVersionResolverTests.java | 53 +++++++++++ ...ventBasedConfigMapChangeDetectorTests.java | 68 +++++++++++++- ...tEventBasedSecretsChangeDetectorTests.java | 69 +++++++++++++- ...nfigurationWatcherHAAutoConfiguration.java | 12 ++- .../ha/ConfigurationWatcherHACoordinator.java | 23 +++-- .../watcher/ha/ConfigurationWatcherState.java | 12 ++- .../ha/ConfigurationWatcherStateStore.java | 14 ++- .../LeaseConfigurationWatcherStateStore.java | 94 +++++++++++-------- ...rationWatcherHAAutoConfigurationTests.java | 91 ++++++++++++++++++ ...onfigurationWatcherHACoordinatorTests.java | 58 ++++++------ ...seConfigurationWatcherStateStoreTests.java | 23 ++--- 14 files changed, 515 insertions(+), 120 deletions(-) create mode 100644 spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/InformerResourceVersionResolver.java create mode 100644 spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/InformerResourceVersionResolverTests.java create mode 100644 spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfigurationTests.java diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/InformerResourceVersionResolver.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/InformerResourceVersionResolver.java new file mode 100644 index 0000000000..214cc9b703 --- /dev/null +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/InformerResourceVersionResolver.java @@ -0,0 +1,74 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.config.reload; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.springframework.core.log.LogAccessor; + +/** + * Resolves the resource version for each informer request. + * + *

+ * When HA is disabled, the resolver always returns the resource version supplied by the + * informer. When HA is enabled, it returns the persisted resource version exactly once, + * for the first request, so that the informer can replay events from the stored + * checkpoint. Every subsequent request must use the resource version supplied by the + * informer because that value advances as the informer processes list and watch calls. + * + * @author wind57 + */ +final class InformerResourceVersionResolver { + + private static final LogAccessor LOG = new LogAccessor(InformerResourceVersionResolver.class); + + // resource versions that we have in our custom lease, these are per namespace + private final Map checkpointResourceVersions; + + private final boolean haEnabled; + + // key is the namespace, value is whether its stored resource + // version was already consumed or not + private final Map checkpointResourceVersionConsumed = new ConcurrentHashMap<>(); + + InformerResourceVersionResolver(Map checkpointResourceVersions, boolean haEnabled) { + this.checkpointResourceVersions = checkpointResourceVersions; + this.haEnabled = haEnabled; + } + + String resolve(String namespace, String informerResourceVersion) { + String checkpointResourceVersion = checkpointResourceVersions.get(namespace); + // If HA is not enabled, do not restore from a checkpoint. + // No persisted checkpoint exists when resourceVersion is null. + if (!haEnabled || checkpointResourceVersion == null) { + return informerResourceVersion; + } + + // Consume the checkpoint once; subsequent versions come from the informer so it + // can progress. + if (checkpointResourceVersionConsumed.computeIfAbsent(namespace, ignored -> new AtomicBoolean()) + .compareAndSet(false, true)) { + LOG.info(() -> "replaying events in namespace " + namespace + " from resource version " + + checkpointResourceVersion); + return checkpointResourceVersion; + } + return informerResourceVersion; + } + +} diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java index b9a051a5d7..e75fbd3e6c 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java @@ -138,14 +138,16 @@ void inform() { // In HA mode, defer informer startup until this instance acquires leadership. // The leader callback restores the persisted state and then starts the informers. if (!haEnabled) { - start(); + start(Map.of()); } } - public final void start() { + public final void start(Map storedResourceVersions) { if (running || !monitoringConfigMaps) { return; } + InformerResourceVersionResolver resourceVersionResolver = new InformerResourceVersionResolver( + storedResourceVersions, haEnabled); LOG.info(() -> "Kubernetes event-based configMap change detector activated"); @@ -166,19 +168,18 @@ public final void start() { namespaces.forEach(namespace -> { SharedIndexInformer informer; - SharedInformerFactory factory = new SharedInformerFactory(apiClient); factories.add(factory); informer = factory - .sharedIndexInformerFor((CallGeneratorParams params) -> coreV1Api.listNamespacedConfigMap(namespace) - .timeoutSeconds(params.timeoutSeconds) - .resourceVersion(params.resourceVersion) - .watch(params.watch) - .labelSelector(labelSelector(labelSelector)) - .buildCall(null), V1ConfigMap.class, V1ConfigMapList.class); - - LOG.debug(() -> "added configmap informer for namespace : " + namespace + " with labels : " - + labelSelector); + .sharedIndexInformerFor((CallGeneratorParams params) -> coreV1Api.listNamespacedConfigMap(namespace) + .timeoutSeconds(params.timeoutSeconds) + .resourceVersion(resourceVersionResolver.resolve(namespace, params.resourceVersion)) + .watch(params.watch) + .labelSelector(labelSelector(labelSelector)) + .buildCall(null), V1ConfigMap.class, V1ConfigMapList.class); + + LOG.debug( + () -> "added configmap informer for namespace : " + namespace + " with labels : " + labelSelector); informer.addEventHandler(handler); informers.add(informer); diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java index 7cab05764b..26da65ffda 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java @@ -142,14 +142,16 @@ void inform() { // In HA mode, defer informer startup until this instance acquires leadership. // The leader callback restores the persisted state and then starts the informers. if (!haEnabled) { - start(); + start(Map.of()); } } - public final void start() { + public final void start(Map storedResourceVersions) { if (running || !monitoringSecrets) { return; } + InformerResourceVersionResolver resourceVersionResolver = new InformerResourceVersionResolver( + storedResourceVersions, haEnabled); LOG.info(() -> "Kubernetes event-based secrets change detector activated"); Map labelSelector; @@ -169,16 +171,15 @@ public final void start() { namespaces.forEach(namespace -> { SharedIndexInformer informer; - SharedInformerFactory factory = new SharedInformerFactory(apiClient); factories.add(factory); informer = factory - .sharedIndexInformerFor((CallGeneratorParams params) -> coreV1Api.listNamespacedSecret(namespace) - .timeoutSeconds(params.timeoutSeconds) - .resourceVersion(params.resourceVersion) - .watch(params.watch) - .labelSelector(labelSelector(labelSelector)) - .buildCall(null), V1Secret.class, V1SecretList.class); + .sharedIndexInformerFor((CallGeneratorParams params) -> coreV1Api.listNamespacedSecret(namespace) + .timeoutSeconds(params.timeoutSeconds) + .resourceVersion(resourceVersionResolver.resolve(namespace, params.resourceVersion)) + .watch(params.watch) + .labelSelector(labelSelector(labelSelector)) + .buildCall(null), V1Secret.class, V1SecretList.class); LOG.debug(() -> "secret informer for namespace : " + namespace + " with filter : " + secretsLabels); diff --git a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/InformerResourceVersionResolverTests.java b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/InformerResourceVersionResolverTests.java new file mode 100644 index 0000000000..753883febb --- /dev/null +++ b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/InformerResourceVersionResolverTests.java @@ -0,0 +1,53 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.config.reload; + +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class InformerResourceVersionResolverTests { + + @Test + void returnsInformerResourceVersionWhenHaIsDisabled() { + InformerResourceVersionResolver resolver = new InformerResourceVersionResolver(Map.of("default", "10"), false); + + assertThat(resolver.resolve("default", "20")).isEqualTo("20"); + } + + @Test + void consumesStoredResourceVersionOnlyOncePerNamespace() { + InformerResourceVersionResolver resolver = new InformerResourceVersionResolver(Map.of("default", "10"), true); + + assertThat(resolver.resolve("default", null)).isEqualTo("10"); + assertThat(resolver.resolve("default", "20")).isEqualTo("20"); + } + + @Test + void consumesStoredResourceVersionIndependentlyForEachNamespace() { + InformerResourceVersionResolver resolver = new InformerResourceVersionResolver( + Map.of("namespace-one", "10", "namespace-two", "20"), true); + + assertThat(resolver.resolve("namespace-one", null)).isEqualTo("10"); + assertThat(resolver.resolve("namespace-two", null)).isEqualTo("20"); + assertThat(resolver.resolve("namespace-one", "11")).isEqualTo("11"); + assertThat(resolver.resolve("namespace-two", "21")).isEqualTo("21"); + } + +} diff --git a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java index 3d7e5b5a8d..438635a807 100644 --- a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java +++ b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java @@ -45,8 +45,10 @@ import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; import static io.kubernetes.client.informer.EventType.ADDED; import static io.kubernetes.client.informer.EventType.DELETED; @@ -219,6 +221,70 @@ void watchStartsOnlyAfterExplicitStartInHaMode() { changeDetectorAssert(true); } + /** + * 

+	 *     - HA mode starts the informer from the persisted resource version for the namespace.
+	 *     - after the initial list, the informer uses the resource version returned by Kubernetes.
+	 *
+ */ + @Test + void watchStartsFromStoredResourceVersionAndThenUsesInformerResourceVersion() { + + // 1. initial request with 'watch=false' and 'resourceVersion=17' + // returns nothing really ( just a dummy list with resourceVersion = 42 ) + V1ConfigMapList configMapList = new V1ConfigMapList().metadata(new V1ListMeta().resourceVersion("42")) + .items(List.of()); + stubFor(get(urlMatching("^/api/v1/namespaces/default/configmaps.*")).withQueryParam("watch", equalTo("false")) + .withQueryParam("resourceVersion", equalTo("17")) + .willReturn(aResponse().withStatus(200).withBody(JSON.serialize(configMapList)))); + + // 2. next request is with 'watch=true' and 'resourceVersion=42', so it's a + // follow-up of + // the next watcher request, it returns a configmap with resourceVersion=43 + V1ConfigMap configMap = new V1ConfigMap() + .metadata(new V1ObjectMeta().namespace("default").name("my-configmap").resourceVersion("43")) + .data(Map.of()); + Response watchResponse = new Response<>(MODIFIED.name(), configMap); + stubFor(get(urlMatching("^/api/v1/namespaces/default/configmaps.*")).withQueryParam("watch", equalTo("true")) + .withQueryParam("resourceVersion", equalTo("42")) + .willReturn(aResponse().withStatus(200).withBody(JSON.serialize(watchResponse)))); + + ApiClient apiClient = new ClientBuilder().setBasePath("http://localhost:" + wireMockServer.port()).build(); + CoreV1Api coreV1Api = new CoreV1Api(apiClient); + int[] onEventCalls = new int[1]; + + KubernetesMockEnvironment environment = new KubernetesMockEnvironment( + mock(KubernetesClientConfigMapPropertySource.class)); + KubernetesClientConfigMapPropertySourceLocator locator = mock( + KubernetesClientConfigMapPropertySourceLocator.class); + + // .withProperty("debug", "false") is needed so that ConfigReloadUtil::reload + // detects a change + when(locator.locate(environment)).thenAnswer(x -> new MockPropertySource().withProperty("debug", "false")); + + // ConfigReloadUtil calls the 'reloadProcedure' from below and we assert its + // invocation + ConfigurationUpdateStrategy strategy = new ConfigurationUpdateStrategy("strategy", () -> ++onEventCalls[0]); + + KubernetesNamespaceProvider namespaceProvider = mock(KubernetesNamespaceProvider.class); + when(namespaceProvider.getNamespace()).thenReturn("default"); + + KubernetesClientEventBasedConfigMapChangeDetector changeDetector = new KubernetesClientEventBasedConfigMapChangeDetector( + coreV1Api, environment, ConfigReloadProperties.DEFAULT, strategy, locator, namespaceProvider, true); + + changeDetector.start(Map.of("default", "17")); + + Awaitilities.awaitUntil(10, 1000, () -> onEventCalls[0] == 1); + verify(getRequestedFor(urlMatching("^/api/v1/namespaces/default/configmaps.*")) + .withQueryParam("watch", equalTo("false")) + .withQueryParam("resourceVersion", equalTo("17"))); + verify(getRequestedFor(urlMatching("^/api/v1/namespaces/default/configmaps.*")) + .withQueryParam("watch", equalTo("true")) + .withQueryParam("resourceVersion", equalTo("42"))); + + changeDetector.shutdown(); + } + private void changeDetectorAssert(boolean haEnabled) { // coreV1Api @@ -252,7 +318,7 @@ private void changeDetectorAssert(boolean haEnabled) { if (haEnabled) { assertThat(onEventCalls[0]).isZero(); - changeDetector.start(); + changeDetector.start(Map.of()); } // all 4 events are caught diff --git a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java index 381897f2f2..4d6a5be9b0 100644 --- a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java +++ b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java @@ -49,8 +49,10 @@ import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; import static io.kubernetes.client.informer.EventType.ADDED; import static io.kubernetes.client.informer.EventType.DELETED; @@ -217,6 +219,71 @@ void watchStartsOnlyAfterExplicitStartInHaMode() { changeDetectorAssert(true); } + /** + * 
+	 *     - HA mode starts the informer from the persisted resource version for the namespace.
+	 *     - after the initial list, the informer uses the resource version returned by Kubernetes.
+	 *
+ */ + @Test + void watchStartsFromStoredResourceVersionAndThenUsesInformerResourceVersion() { + + // 1. initial request with 'watch=false' and 'resourceVersion=17' + // returns nothing really ( just a dummy list with resourceVersion = 42 ) + V1SecretList secretList = new V1SecretList().metadata(new V1ListMeta().resourceVersion("42")).items(List.of()); + stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).withQueryParam("watch", equalTo("false")) + .withQueryParam("resourceVersion", equalTo("17")) + .willReturn(aResponse().withStatus(200).withBody(JSON.serialize(secretList)))); + + // 2. next request is with 'watch=true' and 'resourceVersion=42', so it's a + // follow-up of + // the next watcher request, it returns a secret with resourceVersion=43 + V1Secret secret = new V1Secret() + .metadata(new V1ObjectMeta().namespace("default").name("db-password").resourceVersion("43")) + .data(Map.of()); + Response watchResponse = new Response<>(MODIFIED.name(), secret); + stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).withQueryParam("watch", equalTo("true")) + .withQueryParam("resourceVersion", equalTo("42")) + .willReturn(aResponse().withStatus(200).withBody(JSON.serialize(watchResponse)))); + + ApiClient apiClient = new ClientBuilder().setBasePath("http://localhost:" + wireMockServer.port()).build(); + CoreV1Api coreV1Api = new CoreV1Api(apiClient); + int[] onEventCalls = new int[1]; + + KubernetesMockEnvironment environment = new KubernetesMockEnvironment( + mock(KubernetesClientSecretsPropertySource.class)); + KubernetesClientSecretsPropertySourceLocator locator = mock(KubernetesClientSecretsPropertySourceLocator.class); + + // .withProperty("debug", "false") is needed so that ConfigReloadUtil::reload + // detects a change + when(locator.locate(environment)).thenAnswer(x -> new MockPropertySource().withProperty("debug", "false")); + + // ConfigReloadUtil calls the 'reloadProcedure' from below and we assert its + // invocation + ConfigurationUpdateStrategy strategy = new ConfigurationUpdateStrategy("strategy", () -> ++onEventCalls[0]); + + ConfigReloadProperties properties = new ConfigReloadProperties(false, false, true, + ConfigReloadProperties.ReloadStrategy.REFRESH, ConfigReloadProperties.ReloadDetectionMode.EVENT, + Duration.ofMillis(15000), Set.of(), false, Duration.ofSeconds(2)); + KubernetesNamespaceProvider namespaceProvider = mock(KubernetesNamespaceProvider.class); + when(namespaceProvider.getNamespace()).thenReturn("default"); + + KubernetesClientEventBasedSecretsChangeDetector changeDetector = new KubernetesClientEventBasedSecretsChangeDetector( + coreV1Api, environment, properties, strategy, locator, namespaceProvider, true); + + changeDetector.start(Map.of("default", "17")); + + Awaitilities.awaitUntil(10, 1000, () -> onEventCalls[0] == 1); + verify(getRequestedFor(urlMatching("/api/v1/namespaces/default/secrets.*")) + .withQueryParam("watch", equalTo("false")) + .withQueryParam("resourceVersion", equalTo("17"))); + verify(getRequestedFor(urlMatching("/api/v1/namespaces/default/secrets.*")) + .withQueryParam("watch", equalTo("true")) + .withQueryParam("resourceVersion", equalTo("42"))); + + changeDetector.shutdown(); + } + /** * both are null, treat that as no change. */ @@ -368,7 +435,7 @@ private void changeDetectorAssert(boolean haEnabled) { if (haEnabled) { Assertions.assertThat(onEventCalls[0]).isZero(); - changeDetector.start(); + changeDetector.start(Map.of()); } // all 4 events are caught diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfiguration.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfiguration.java index 08a1a1fb4b..16447f1343 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfiguration.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfiguration.java @@ -26,6 +26,7 @@ import org.springframework.boot.cloud.CloudPlatform; import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedConfigMapChangeDetector; import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedSecretsChangeDetector; +import org.springframework.cloud.kubernetes.commons.leader.election.ConditionalOnLeaderElectionEnabled; import org.springframework.cloud.kubernetes.configuration.watcher.ConfigurationWatcherConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -33,14 +34,16 @@ /** * Configures the components required for configuration watcher HA. * - *

Creates the persistent state store and the coordinator that starts and stops - * the watchers when leadership changes. + *

+ * Creates the persistent state store and the coordinator that starts and stops the + * watchers when leadership changes. * * @author wind57 */ @Configuration(proxyBeanMethods = false) @ConditionalOnCloudPlatform(CloudPlatform.KUBERNETES) @ConditionalOnConfigurationWatcherHAEnabled +@ConditionalOnLeaderElectionEnabled @ConditionalOnBean(ApiClient.class) class ConfigurationWatcherHAAutoConfiguration { @@ -55,8 +58,9 @@ ConfigurationWatcherStateStore configurationWatcherStateStore(ApiClient apiClien @ConditionalOnMissingBean ConfigurationWatcherHACoordinator configurationWatcherHACoordinator( ObjectProvider configMapDetector, - ObjectProvider secretsDetector) { - return new ConfigurationWatcherHACoordinator(configMapDetector, secretsDetector); + ObjectProvider secretsDetector, + ConfigurationWatcherStateStore stateStore) { + return new ConfigurationWatcherHACoordinator(configMapDetector, secretsDetector, stateStore); } } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java index 744adcf5c0..af599515d1 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java @@ -39,29 +39,34 @@ public final class ConfigurationWatcherHACoordinator { private final ObjectProvider secretsDetector; + private final ConfigurationWatcherStateStore stateStore; + public ConfigurationWatcherHACoordinator( ObjectProvider configMapDetector, - ObjectProvider secretsDetector) { + ObjectProvider secretsDetector, + ConfigurationWatcherStateStore stateStore) { if (configMapDetector.getIfAvailable() == null && secretsDetector.getIfAvailable() == null) { - throw new IllegalStateException("Configuration watcher HA is enabled, but neither ConfigMap nor Secret " - + "watching is enabled"); + throw new IllegalStateException( + "Configuration watcher HA is enabled, but neither ConfigMap nor Secret " + "watching is enabled"); } this.configMapDetector = configMapDetector; this.secretsDetector = secretsDetector; + this.stateStore = stateStore; } @EventListener void onStartLeading(StartLeadingEvent event) { - LOG.info(() -> "configuration watcher with identity : " + event.candidateIdentity() + - " became leader at : " + Instant.ofEpochMilli(event.getTimestamp())); - configMapDetector.ifAvailable(KubernetesClientEventBasedConfigMapChangeDetector::start); - secretsDetector.ifAvailable(KubernetesClientEventBasedSecretsChangeDetector::start); + LOG.info(() -> "configuration watcher with identity : " + event.candidateIdentity() + " became leader at : " + + Instant.ofEpochMilli(event.getTimestamp())); + ConfigurationWatcherState state = stateStore.readOrCreate(); + configMapDetector.ifAvailable(detector -> detector.start(state.configMapResourceVersions())); + secretsDetector.ifAvailable(detector -> detector.start(state.secretResourceVersions())); } @EventListener void onStopLeading(StopLeadingEvent event) { - LOG.info(() -> "configuration watcher with identity : " + event.candidateIdentity() + - " stopped being a leader at : " + Instant.ofEpochMilli(event.getTimestamp())); + LOG.info(() -> "configuration watcher with identity : " + event.candidateIdentity() + + " stopped being a leader at : " + Instant.ofEpochMilli(event.getTimestamp())); secretsDetector.ifAvailable(KubernetesClientEventBasedSecretsChangeDetector::stop); configMapDetector.ifAvailable(KubernetesClientEventBasedConfigMapChangeDetector::stop); } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherState.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherState.java index 749be2f1a2..a8615cbf06 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherState.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherState.java @@ -16,15 +16,19 @@ package org.springframework.cloud.kubernetes.configuration.watcher.ha; +import java.util.Map; + /** * Persisted HA watcher state. * - * @param configMapResourceVersion last processed configmap resource version - * @param secretResourceVersion last processed secret resource version + * @param configMapResourceVersions last processed ConfigMap resource version per + * namespace + * @param secretResourceVersions last processed Secret resource version per namespace * @author wind57 */ -record ConfigurationWatcherState(String configMapResourceVersion, String secretResourceVersion) { +record ConfigurationWatcherState(Map configMapResourceVersions, + Map secretResourceVersions) { - static final ConfigurationWatcherState EMPTY = new ConfigurationWatcherState(null, null); + static final ConfigurationWatcherState EMPTY = new ConfigurationWatcherState(Map.of(), Map.of()); } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherStateStore.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherStateStore.java index 3c49f4b79a..6f6ebc93cb 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherStateStore.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherStateStore.java @@ -23,8 +23,20 @@ */ sealed interface ConfigurationWatcherStateStore permits LeaseConfigurationWatcherStateStore { + /** + * Reads or creates the single persisted state used by the configuration watcher. + * + *

+ * When HA is enabled, the leader calls this once before starting the ConfigMap and + * Secret informers. The returned state contains checkpoints for all configured + * namespaces, so this method must not be called once per namespace. It may be called + * again if a later leadership acquisition starts the watcher again. + * @return the persisted state, or an empty state when no checkpoint exists + */ ConfigurationWatcherState readOrCreate(); - void write(ConfigurationWatcherState state); + void writeConfigMapResourceVersion(String namespace, String resourceVersion); + + void writeSecretResourceVersion(String namespace, String resourceVersion); } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java index 4174b0e23b..aab19c9247 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java @@ -17,7 +17,9 @@ package org.springframework.cloud.kubernetes.configuration.watcher.ha; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; +import java.util.stream.Collectors; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.apis.CoordinationV1Api; @@ -41,59 +43,67 @@ final class LeaseConfigurationWatcherStateStore implements ConfigurationWatcherS private static final LogAccessor LOG = new LogAccessor(LeaseConfigurationWatcherStateStore.class); - private static final String CONFIGMAP_ANNOTATION = "spring.cloud.kubernetes.configuration.watcher.configmap-resource-version"; + private static final String CONFIGMAP_ANNOTATION = "spring.cloud.kubernetes.configuration.watcher/configmap-resource-version"; - private static final String SECRET_ANNOTATION = "spring.cloud.kubernetes.configuration.watcher.secret-resource-version"; + private static final String SECRET_ANNOTATION = "spring.cloud.kubernetes.configuration.watcher/secret-resource-version"; private final CoordinationV1Api api; - private final ConfigurationWatcherHaProperties properties; + private final String leaseName; + + private final String leaseNamespace; LeaseConfigurationWatcherStateStore(CoordinationV1Api api, ConfigurationWatcherHaProperties properties) { this.api = api; - this.properties = properties; + this.leaseName = properties.getLeaseName(); + String configuredLeaseNamespace = properties.getLeaseNamespace(); + this.leaseNamespace = StringUtils.hasText(configuredLeaseNamespace) ? configuredLeaseNamespace : "default"; } @Override public ConfigurationWatcherState readOrCreate() { - String leaseName = properties.getLeaseName(); - String leaseNamespace = namespace(); - LOG.debug("Reading lease with name: " + leaseName + " in namespace: " + leaseNamespace); + LOG.debug(() -> "Reading lease with name: " + leaseName + " in namespace: " + leaseNamespace); try { V1Lease lease = api.readNamespacedLease(leaseName, leaseNamespace).execute(); Map annotations = existingAnnotations(lease); - return new ConfigurationWatcherState(annotations.get(CONFIGMAP_ANNOTATION), - annotations.get(SECRET_ANNOTATION)); + return new ConfigurationWatcherState(parseResourceVersions(annotations.get(CONFIGMAP_ANNOTATION)), + parseResourceVersions(annotations.get(SECRET_ANNOTATION))); } catch (ApiException e) { if (e.getCode() == 404) { createLease(leaseName, leaseNamespace); return ConfigurationWatcherState.EMPTY; } - throw new IllegalStateException("Failed to read watcher HA lease '" + properties.getLeaseName() - + "' in namespace '" + leaseNamespace + "'", e); + throw new IllegalStateException( + "Failed to read watcher HA lease '" + leaseName + "' in namespace '" + leaseNamespace + "'", e); } } @Override - public void write(ConfigurationWatcherState state) { + public void writeConfigMapResourceVersion(String namespace, String resourceVersion) { + writeResourceVersion(CONFIGMAP_ANNOTATION, namespace, resourceVersion); + } + + @Override + public void writeSecretResourceVersion(String namespace, String resourceVersion) { + writeResourceVersion(SECRET_ANNOTATION, namespace, resourceVersion); + } + + private void writeResourceVersion(String annotation, String namespace, String resourceVersion) { try { - String leaseName = properties.getLeaseName(); - String leaseNamespace = namespace(); - LOG.debug("Updating lease with name: " + leaseName + " in namespace: " + leaseNamespace); + LOG.debug(() -> "Updating lease with name: " + leaseName + " in namespace: " + leaseNamespace); V1Lease currentLease = api.readNamespacedLease(leaseName, leaseNamespace).execute(); - // add the annotations - V1Lease updatedLease = updatedLease(currentLease, state); + V1Lease updatedLease = updatedLease(currentLease, annotation, namespace, resourceVersion); api.replaceNamespacedLease(leaseName, leaseNamespace, updatedLease).execute(); } catch (ApiException e) { LOG.error(e, () -> "Failed to write to the lease because : " + e.getResponseBody()); - throw new IllegalStateException("Failed to write watcher HA lease '" + properties.getLeaseName() - + "' in namespace '" + namespace() + "'", e); + throw new IllegalStateException( + "Failed to write watcher HA lease '" + leaseName + "' in namespace '" + leaseNamespace + "'", e); } } @@ -104,15 +114,19 @@ private void createLease(String leaseName, String leaseNamespace) { } catch (ApiException e) { LOG.error(e, () -> "Failed to create watcher HA lease '" + e.getResponseBody()); - throw new IllegalStateException("Failed to create watcher HA lease '" + properties.getLeaseName() - + "' in namespace '" + namespace() + "'", e); + throw new IllegalStateException( + "Failed to create watcher HA lease '" + leaseName + "' in namespace '" + leaseNamespace + "'", e); } } - private V1Lease updatedLease(V1Lease lease, ConfigurationWatcherState state) { + private V1Lease updatedLease(V1Lease lease, String annotation, String namespace, String resourceVersion) { V1ObjectMeta metadata = lease.getMetadata(); Map currentLeaseAnnotations = existingAnnotations(lease); - metadata.setAnnotations(updateAnnotations(currentLeaseAnnotations, state)); + Map updatedAnnotations = new HashMap<>(currentLeaseAnnotations); + Map resourceVersions = parseResourceVersions(updatedAnnotations.get(annotation)); + resourceVersions.put(namespace, resourceVersion); + updatedAnnotations.put(annotation, serializeResourceVersions(resourceVersions)); + metadata.setAnnotations(updatedAnnotations); return lease; } @@ -131,26 +145,28 @@ private static Map existingAnnotations(V1Lease lease) { return lease.getMetadata().getAnnotations(); } - private static Map updateAnnotations(Map existingAnnotations, - ConfigurationWatcherState state) { - Map annotations = new HashMap<>(existingAnnotations); + private static String serializeResourceVersions(Map resourceVersions) { + return resourceVersions.entrySet() + .stream() + .sorted(Map.Entry.comparingByKey()) + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(Collectors.joining(",")); + } - String configMapResourceVersion = state.configMapResourceVersion(); - if (configMapResourceVersion != null) { - annotations.put(CONFIGMAP_ANNOTATION, configMapResourceVersion); + private static Map parseResourceVersions(String serializedResourceVersions) { + if (!StringUtils.hasText(serializedResourceVersions)) { + return Map.of(); } - String secretResourceVersion = state.secretResourceVersion(); - if (secretResourceVersion != null) { - annotations.put(SECRET_ANNOTATION, secretResourceVersion); + Map resourceVersions = new LinkedHashMap<>(); + for (String entry : serializedResourceVersions.split(",")) { + String[] keyValue = entry.split("=", 2); + if (keyValue.length != 2) { + throw new IllegalStateException("Invalid ConfigMap resource version entry: " + entry); + } + resourceVersions.put(keyValue[0], keyValue[1]); } - - return annotations; - } - - private String namespace() { - String leaseNamespaceFromProperties = properties.getLeaseNamespace(); - return StringUtils.hasText(leaseNamespaceFromProperties) ? leaseNamespaceFromProperties : "default"; + return resourceVersions; } } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfigurationTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfigurationTests.java new file mode 100644 index 0000000000..95d7758fbd --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfigurationTests.java @@ -0,0 +1,91 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher.ha; + +import io.kubernetes.client.openapi.ApiClient; +import org.junit.jupiter.api.Test; + +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedConfigMapChangeDetector; +import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedSecretsChangeDetector; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +/** + * @author wind57 + */ +class ConfigurationWatcherHAAutoConfigurationTests { + + @Test + void createsCoordinatorWhenHaAndLeaderElectionAreEnabled() { + applicationContextRunner() + .withPropertyValues("spring.cloud.kubernetes.configuration.watcher.ha.enabled=true", + "spring.cloud.kubernetes.leader.election.enabled=true") + .run(context -> assertThat(context).hasSingleBean(ConfigurationWatcherHACoordinator.class)); + } + + @Test + void doesNotCreateCoordinatorWhenLeaderElectionIsDisabled() { + applicationContextRunner() + .withPropertyValues("spring.cloud.kubernetes.configuration.watcher.ha.enabled=true", + "spring.cloud.kubernetes.leader.election.enabled=false") + .run(context -> assertThat(context).doesNotHaveBean(ConfigurationWatcherHACoordinator.class)); + } + + @Test + void doesNotCreateCoordinatorWhenHaIsDisabled() { + applicationContextRunner() + .withPropertyValues("spring.cloud.kubernetes.configuration.watcher.ha.enabled=false", + "spring.cloud.kubernetes.leader.election.enabled=true") + .run(context -> assertThat(context).doesNotHaveBean(ConfigurationWatcherHACoordinator.class)); + } + + private ApplicationContextRunner applicationContextRunner() { + return new ApplicationContextRunner() + .withUserConfiguration(TestConfiguration.class, ConfigurationWatcherHAAutoConfiguration.class) + .withPropertyValues("spring.main.cloud-platform=KUBERNETES"); + } + + @Configuration(proxyBeanMethods = false) + static class TestConfiguration { + + @Bean + ApiClient apiClient() { + return mock(ApiClient.class); + } + + @Bean + ConfigurationWatcherStateStore stateStore() { + return mock(ConfigurationWatcherStateStore.class); + } + + @Bean + KubernetesClientEventBasedConfigMapChangeDetector configMapDetector() { + return mock(KubernetesClientEventBasedConfigMapChangeDetector.class); + } + + @Bean + KubernetesClientEventBasedSecretsChangeDetector secretsDetector() { + return mock(KubernetesClientEventBasedSecretsChangeDetector.class); + } + + } + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java index ae9131918c..aa09234443 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java @@ -16,10 +16,11 @@ package org.springframework.cloud.kubernetes.configuration.watcher.ha; +import java.util.Map; + import org.junit.jupiter.api.Test; import org.springframework.beans.factory.ObjectProvider; -import org.springframework.beans.factory.support.DefaultListableBeanFactory; import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedConfigMapChangeDetector; import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedSecretsChangeDetector; import org.springframework.cloud.kubernetes.commons.leader.election.events.StartLeadingEvent; @@ -28,6 +29,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * @author wind57 @@ -42,19 +44,23 @@ void onStartLeadingStartsBothDetectors() { KubernetesClientEventBasedSecretsChangeDetector secretsDetector = mock( KubernetesClientEventBasedSecretsChangeDetector.class); - ObjectProvider configMapProvider = provider( - KubernetesClientEventBasedConfigMapChangeDetector.class, configMapDetector); + ObjectProvider configMapProvider = mock( + ObjectProvider.class); + when(configMapProvider.getIfAvailable()).thenReturn(configMapDetector); - ObjectProvider secretsProvider = provider( - KubernetesClientEventBasedSecretsChangeDetector.class, secretsDetector); + ObjectProvider secretsProvider = mock(ObjectProvider.class); + when(secretsProvider.getIfAvailable()).thenReturn(secretsDetector); + ConfigurationWatcherStateStore stateStore = mock(ConfigurationWatcherStateStore.class); + when(stateStore.readOrCreate()).thenReturn( + new ConfigurationWatcherState(Map.of("default", "config-map-rv"), Map.of("default", "secret-rv"))); ConfigurationWatcherHACoordinator coordinator = new ConfigurationWatcherHACoordinator(configMapProvider, - secretsProvider); + secretsProvider, stateStore); coordinator.onStartLeading(new StartLeadingEvent("candidate")); - verify(configMapDetector).start(); - verify(secretsDetector).start(); + verify(configMapDetector).start(Map.of("default", "config-map-rv")); + verify(secretsDetector).start(Map.of("default", "secret-rv")); } @Test @@ -65,14 +71,16 @@ void onStopLeadingStopsBothDetectors() { KubernetesClientEventBasedSecretsChangeDetector secretsDetector = mock( KubernetesClientEventBasedSecretsChangeDetector.class); - ObjectProvider configMapProvider = provider( - KubernetesClientEventBasedConfigMapChangeDetector.class, configMapDetector); + ObjectProvider configMapProvider = mock( + ObjectProvider.class); + when(configMapProvider.getIfAvailable()).thenReturn(configMapDetector); - ObjectProvider secretsProvider = provider( - KubernetesClientEventBasedSecretsChangeDetector.class, secretsDetector); + ObjectProvider secretsProvider = mock(ObjectProvider.class); + when(secretsProvider.getIfAvailable()).thenReturn(secretsDetector); + ConfigurationWatcherStateStore stateStore = mock(ConfigurationWatcherStateStore.class); ConfigurationWatcherHACoordinator coordinator = new ConfigurationWatcherHACoordinator(configMapProvider, - secretsProvider); + secretsProvider, stateStore); coordinator.onStopLeading(new StopLeadingEvent("candidate")); @@ -82,24 +90,16 @@ void onStopLeadingStopsBothDetectors() { @Test void failsWhenNeitherDetectorIsAvailable() { - ObjectProvider configMapProvider = emptyProvider( - KubernetesClientEventBasedConfigMapChangeDetector.class); - ObjectProvider secretsProvider = emptyProvider( - KubernetesClientEventBasedSecretsChangeDetector.class); - - assertThatThrownBy(() -> new ConfigurationWatcherHACoordinator(configMapProvider, secretsProvider)) + ObjectProvider configMapProvider = mock( + ObjectProvider.class); + when(configMapProvider.getIfAvailable()).thenReturn(null); + ObjectProvider secretsProvider = mock(ObjectProvider.class); + when(secretsProvider.getIfAvailable()).thenReturn(null); + ConfigurationWatcherStateStore stateStore = mock(ConfigurationWatcherStateStore.class); + + assertThatThrownBy(() -> new ConfigurationWatcherHACoordinator(configMapProvider, secretsProvider, stateStore)) .isInstanceOf(IllegalStateException.class) .hasMessage("Configuration watcher HA is enabled, but neither ConfigMap nor Secret watching is enabled"); } - private static ObjectProvider provider(Class type, T value) { - DefaultListableBeanFactory beanFactory = new DefaultListableBeanFactory(); - beanFactory.registerSingleton("detector", value); - return beanFactory.getBeanProvider(type); - } - - private static ObjectProvider emptyProvider(Class type) { - return new DefaultListableBeanFactory().getBeanProvider(type); - } - } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStoreTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStoreTests.java index 7c8f5c8b83..8f4d68e107 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStoreTests.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStoreTests.java @@ -58,9 +58,9 @@ class LeaseConfigurationWatcherStateStoreTests { private static final String LEASE_COLLECTION_URL = "/apis/coordination.k8s.io/v1/namespaces/" + LEASE_NAMESPACE + "/leases"; - private static final String CONFIGMAP_RESOURCE_VERSION_ANNOTATION = "spring.cloud.kubernetes.configuration.watcher.configmap-resource-version"; + private static final String CONFIGMAP_RESOURCE_VERSION_ANNOTATION = "spring.cloud.kubernetes.configuration.watcher/configmap-resource-version"; - private static final String SECRET_RESOURCE_VERSION_ANNOTATION = "spring.cloud.kubernetes.configuration.watcher.secret-resource-version"; + private static final String SECRET_RESOURCE_VERSION_ANNOTATION = "spring.cloud.kubernetes.configuration.watcher/secret-resource-version"; private static WireMockServer wireMockServer; @@ -103,8 +103,8 @@ void readOrCreateReturnsExistingStateFromLeaseAnnotations() { ConfigurationWatcherState state = stateStore.readOrCreate(); - assertThat(state.configMapResourceVersion()).isEqualTo(configMapResourceVersion); - assertThat(state.secretResourceVersion()).isEqualTo(secretResourceVersion); + assertThat(state.configMapResourceVersions()).containsEntry(LEASE_NAMESPACE, configMapResourceVersion); + assertThat(state.secretResourceVersions()).containsEntry(LEASE_NAMESPACE, secretResourceVersion); wireMockServer.verify(getRequestedFor(urlEqualTo(LEASE_URL))); } @@ -150,15 +150,15 @@ void writeUpdatesLeaseAnnotations() { stubFor(get(urlEqualTo(LEASE_URL)).willReturn(aResponse().withStatus(200).withBody(serialize(existingLease)))); stubFor(put(urlEqualTo(LEASE_URL)).willReturn(aResponse().withStatus(200).withBody(serialize(updatedLease)))); - stateStore.write(new ConfigurationWatcherState(configMapResourceVersion, null)); + stateStore.writeConfigMapResourceVersion(LEASE_NAMESPACE, configMapResourceVersion); wireMockServer.verify(getRequestedFor(urlEqualTo(LEASE_URL))); wireMockServer.verify(putRequestedFor(urlEqualTo(LEASE_URL)) .withRequestBody( matchingJsonPath("$.metadata.annotations." + jsonPath(CONFIGMAP_RESOURCE_VERSION_ANNOTATION), - equalTo(configMapResourceVersion))) + equalTo(LEASE_NAMESPACE + "=" + configMapResourceVersion))) .withRequestBody(matchingJsonPath("$.metadata.annotations." + jsonPath(SECRET_RESOURCE_VERSION_ANNOTATION), - equalTo(existingSecretResourceVersion)))); + equalTo(LEASE_NAMESPACE + "=" + existingSecretResourceVersion)))); } /** @@ -188,7 +188,8 @@ void readOrCreateThrowsWhenMissingLeaseCanNotBeCreated() { stubFor(post(urlEqualTo(LEASE_COLLECTION_URL)).willReturn(aResponse().withStatus(500).withBody("boom"))); assertThatThrownBy(() -> stateStore.readOrCreate()).isInstanceOf(IllegalStateException.class) - .hasMessage("Failed to create watcher HA lease '" + LEASE_NAME + "' in namespace '" + LEASE_NAMESPACE + "'"); + .hasMessage( + "Failed to create watcher HA lease '" + LEASE_NAME + "' in namespace '" + LEASE_NAMESPACE + "'"); } /** @@ -208,7 +209,7 @@ void writeThrowsWhenLeaseUpdateFails() { stubFor(get(urlEqualTo(LEASE_URL)).willReturn(aResponse().withStatus(200).withBody(serialize(existingLease)))); stubFor(put(urlEqualTo(LEASE_URL)).willReturn(aResponse().withStatus(500).withBody("boom"))); - assertThatThrownBy(() -> stateStore.write(new ConfigurationWatcherState(configMapResourceVersion, null))) + assertThatThrownBy(() -> stateStore.writeConfigMapResourceVersion(LEASE_NAMESPACE, configMapResourceVersion)) .isInstanceOf(IllegalStateException.class) .hasMessage("Failed to write watcher HA lease '" + LEASE_NAME + "' in namespace '" + LEASE_NAMESPACE + "'"); } @@ -216,10 +217,10 @@ void writeThrowsWhenLeaseUpdateFails() { private static V1Lease leaseWithAnnotations(String configMapResourceVersion, String secretResourceVersion) { Map annotations = new HashMap<>(); if (configMapResourceVersion != null) { - annotations.put(CONFIGMAP_RESOURCE_VERSION_ANNOTATION, configMapResourceVersion); + annotations.put(CONFIGMAP_RESOURCE_VERSION_ANNOTATION, LEASE_NAMESPACE + "=" + configMapResourceVersion); } if (secretResourceVersion != null) { - annotations.put(SECRET_RESOURCE_VERSION_ANNOTATION, secretResourceVersion); + annotations.put(SECRET_RESOURCE_VERSION_ANNOTATION, LEASE_NAMESPACE + "=" + secretResourceVersion); } return new V1Lease() .metadata(new V1ObjectMeta().name(LEASE_NAME).namespace(LEASE_NAMESPACE).annotations(annotations)); From 8db54bf92de2dbf10a9411e5519d3589da3717c6 Mon Sep 17 00:00:00 2001 From: wind57 Date: Tue, 18 Aug 2026 16:28:43 +0300 Subject: [PATCH 08/24] minor refactor Signed-off-by: wind57 --- .../InformerResourceVersionResolver.java | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/InformerResourceVersionResolver.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/InformerResourceVersionResolver.java index 214cc9b703..2efc89927d 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/InformerResourceVersionResolver.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/InformerResourceVersionResolver.java @@ -53,17 +53,26 @@ final class InformerResourceVersionResolver { } String resolve(String namespace, String informerResourceVersion) { + + // if HA is not enabled, we do not restore from a checkpoint + if (!haEnabled) { + return informerResourceVersion; + } + String checkpointResourceVersion = checkpointResourceVersions.get(namespace); - // If HA is not enabled, do not restore from a checkpoint. - // No persisted checkpoint exists when resourceVersion is null. - if (!haEnabled || checkpointResourceVersion == null) { + // there is no previous checkpoint ( maybe it's the first time app is started in + // HA mode) + if (checkpointResourceVersion == null) { return informerResourceVersion; } - // Consume the checkpoint once; subsequent versions come from the informer so it - // can progress. - if (checkpointResourceVersionConsumed.computeIfAbsent(namespace, ignored -> new AtomicBoolean()) - .compareAndSet(false, true)) { + // Consume the checkpoint only once. + // Subsequent versions come from the informer, so it can progress. + boolean checkpointNotConsumed = checkpointResourceVersionConsumed + .computeIfAbsent(namespace, ignored -> new AtomicBoolean()) + .compareAndSet(false, true); + + if (checkpointNotConsumed) { LOG.info(() -> "replaying events in namespace " + namespace + " from resource version " + checkpointResourceVersion); return checkpointResourceVersion; From ae974375914a4e9b8d6692caa913fa49bf23e0af Mon Sep 17 00:00:00 2001 From: wind57 Date: Wed, 19 Aug 2026 18:51:53 +0300 Subject: [PATCH 09/24] added some tests Signed-off-by: wind57 --- .../reload/ConfigMapResourceEventHandler.java | 20 +++++++++++++- ...ientEventBasedConfigMapChangeDetector.java | 11 +++++--- ...ClientEventBasedSecretsChangeDetector.java | 11 +++++--- .../reload/NamespaceAndResourceVersion.java | 27 +++++++++++++++++++ .../reload/SecretResourceEventHandler.java | 20 +++++++++++++- ...ventBasedConfigMapChangeDetectorTests.java | 10 ++++--- ...tEventBasedSecretsChangeDetectorTests.java | 10 ++++--- .../ha/ConfigurationWatcherHACoordinator.java | 15 +++++++++-- ...onfigurationWatcherHACoordinatorTests.java | 6 +++-- 9 files changed, 110 insertions(+), 20 deletions(-) create mode 100644 spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/NamespaceAndResourceVersion.java diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/ConfigMapResourceEventHandler.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/ConfigMapResourceEventHandler.java index f1c07284f0..746771f228 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/ConfigMapResourceEventHandler.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/ConfigMapResourceEventHandler.java @@ -21,6 +21,8 @@ import io.kubernetes.client.informer.ResourceEventHandler; import io.kubernetes.client.openapi.models.V1ConfigMap; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import jakarta.annotation.Nullable; import org.apache.commons.logging.LogFactory; import org.springframework.core.log.LogAccessor; @@ -34,8 +36,13 @@ final class ConfigMapResourceEventHandler implements ResourceEventHandler onEvent; - ConfigMapResourceEventHandler(Consumer onEvent) { + @Nullable + private final Consumer resourceVersionWriter; + + ConfigMapResourceEventHandler(Consumer onEvent, + @Nullable Consumer resourceVersionWriter) { this.onEvent = onEvent; + this.resourceVersionWriter = resourceVersionWriter; } @Override @@ -43,6 +50,7 @@ public void onAdd(V1ConfigMap configMap) { LOG.debug(() -> "ConfigMap " + configMap.getMetadata().getName() + " was added in namespace " + configMap.getMetadata().getNamespace()); onEvent.accept(configMap); + writeResourceVersion(configMap); } @Override @@ -55,6 +63,7 @@ public void onUpdate(V1ConfigMap oldConfigMap, V1ConfigMap newConfigMap) { else { onEvent.accept(newConfigMap); } + writeResourceVersion(newConfigMap); } @Override @@ -62,6 +71,15 @@ public void onDelete(V1ConfigMap configMap, boolean deletedFinalStateUnknown) { LOG.debug(() -> "ConfigMap " + configMap.getMetadata().getName() + " was deleted in namespace " + configMap.getMetadata().getNamespace()); onEvent.accept(configMap); + writeResourceVersion(configMap); + } + + private void writeResourceVersion(V1ConfigMap configMap) { + if (resourceVersionWriter != null) { + V1ObjectMeta metadata = configMap.getMetadata(); + resourceVersionWriter.accept(new NamespaceAndResourceVersion( + metadata.getNamespace(), metadata.getResourceVersion())); + } } } diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java index e8e67a7f7d..41dd7c82af 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Consumer; import io.kubernetes.client.common.KubernetesObject; import io.kubernetes.client.informer.SharedIndexInformer; @@ -29,6 +30,7 @@ import io.kubernetes.client.openapi.models.V1ConfigMap; import io.kubernetes.client.openapi.models.V1ConfigMapList; import io.kubernetes.client.util.CallGeneratorParams; +import jakarta.annotation.Nullable; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; @@ -79,8 +81,6 @@ public class KubernetesClientEventBasedConfigMapChangeDetector extends Configura // informers already running (skip starting more informers) private volatile boolean running; - private final ConfigMapResourceEventHandler handler = new ConfigMapResourceEventHandler(this::onEvent); - public KubernetesClientEventBasedConfigMapChangeDetector(CoreV1Api coreV1Api, ConfigurableEnvironment environment, ConfigReloadProperties properties, ConfigurationUpdateStrategy strategy, KubernetesClientConfigMapPropertySourceLocator propertySourceLocator, @@ -109,11 +109,12 @@ void inform() { // In HA mode, defer informer startup until this instance acquires leadership. // The leader callback restores the persisted state and then starts the informers. if (!haEnabled) { - start(Map.of()); + start(Map.of(), null); } } - public final void start(Map storedResourceVersions) { + public final void start(Map storedResourceVersions, + @Nullable Consumer resourceVersionWriter) { if (running || !monitoringConfigMaps) { return; } @@ -139,6 +140,8 @@ public final void start(Map storedResourceVersions) { namespaces.forEach(namespace -> { SharedIndexInformer informer; + ConfigMapResourceEventHandler handler = new ConfigMapResourceEventHandler(this::onEvent, + resourceVersionWriter); SharedInformerFactory factory = new SharedInformerFactory(apiClient); factories.add(factory); informer = factory diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java index 381a9af495..c45efa3e2b 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Consumer; import io.kubernetes.client.common.KubernetesObject; import io.kubernetes.client.informer.SharedIndexInformer; @@ -31,6 +32,7 @@ import io.kubernetes.client.openapi.models.V1Secret; import io.kubernetes.client.openapi.models.V1SecretList; import io.kubernetes.client.util.CallGeneratorParams; +import jakarta.annotation.Nullable; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import org.apache.commons.logging.LogFactory; @@ -83,8 +85,6 @@ public class KubernetesClientEventBasedSecretsChangeDetector extends Configurati // informers already running (skip starting more informers) private volatile boolean running; - private final SecretResourceEventHandler handler = new SecretResourceEventHandler(LOG, this::onEvent); - public KubernetesClientEventBasedSecretsChangeDetector(CoreV1Api coreV1Api, ConfigurableEnvironment environment, ConfigReloadProperties properties, ConfigurationUpdateStrategy strategy, KubernetesClientSecretsPropertySourceLocator propertySourceLocator, @@ -113,11 +113,12 @@ void inform() { // In HA mode, defer informer startup until this instance acquires leadership. // The leader callback restores the persisted state and then starts the informers. if (!haEnabled) { - start(Map.of()); + start(Map.of(), null); } } - public final void start(Map storedResourceVersions) { + public final void start(Map storedResourceVersions, + @Nullable Consumer resourceVersionWriter) { if (running || !monitoringSecrets) { return; } @@ -142,6 +143,8 @@ public final void start(Map storedResourceVersions) { namespaces.forEach(namespace -> { SharedIndexInformer informer; + SecretResourceEventHandler handler = new SecretResourceEventHandler(LOG, this::onEvent, + resourceVersionWriter); SharedInformerFactory factory = new SharedInformerFactory(apiClient); factories.add(factory); informer = factory diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/NamespaceAndResourceVersion.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/NamespaceAndResourceVersion.java new file mode 100644 index 0000000000..2e6194f8f4 --- /dev/null +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/NamespaceAndResourceVersion.java @@ -0,0 +1,27 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.config.reload; + +/** + * Identifies an informer resource version within a Kubernetes namespace. + * + * @param namespace the Kubernetes namespace + * @param resourceVersion the informer resource version + * @author wind57 + */ +public record NamespaceAndResourceVersion(String namespace, String resourceVersion) { +} diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/SecretResourceEventHandler.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/SecretResourceEventHandler.java index 7ac47b6c3f..71f8f43789 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/SecretResourceEventHandler.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/SecretResourceEventHandler.java @@ -19,8 +19,10 @@ import java.util.function.Consumer; import io.kubernetes.client.informer.ResourceEventHandler; +import io.kubernetes.client.openapi.models.V1ObjectMeta; import io.kubernetes.client.openapi.models.V1Secret; +import jakarta.annotation.Nullable; import org.springframework.core.log.LogAccessor; final class SecretResourceEventHandler implements ResourceEventHandler { @@ -29,9 +31,14 @@ final class SecretResourceEventHandler implements ResourceEventHandler private final Consumer onEvent; - SecretResourceEventHandler(LogAccessor log, Consumer onEvent) { + @Nullable + private final Consumer resourceVersionWriter; + + SecretResourceEventHandler(LogAccessor log, Consumer onEvent, + @Nullable Consumer resourceVersionWriter) { this.log = log; this.onEvent = onEvent; + this.resourceVersionWriter = resourceVersionWriter; } @Override @@ -39,6 +46,7 @@ public void onAdd(V1Secret secret) { log.debug(() -> "Secret " + secret.getMetadata().getName() + " was added in namespace " + secret.getMetadata().getNamespace()); onEvent.accept(secret); + writeResourceVersion(secret); } @Override @@ -52,6 +60,7 @@ public void onUpdate(V1Secret oldSecret, V1Secret newSecret) { else { onEvent.accept(newSecret); } + writeResourceVersion(newSecret); } @Override @@ -59,6 +68,15 @@ public void onDelete(V1Secret secret, boolean deletedFinalStateUnknown) { log.debug(() -> "Secret " + secret.getMetadata().getName() + " was deleted in namespace " + secret.getMetadata().getNamespace()); onEvent.accept(secret); + writeResourceVersion(secret); + } + + private void writeResourceVersion(V1Secret secret) { + if (resourceVersionWriter != null) { + V1ObjectMeta metadata = secret.getMetadata(); + resourceVersionWriter.accept(new NamespaceAndResourceVersion( + metadata.getNamespace(), metadata.getResourceVersion())); + } } } diff --git a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java index 438635a807..29e3f6ef76 100644 --- a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java +++ b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java @@ -16,6 +16,7 @@ package org.springframework.cloud.kubernetes.client.config.reload; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -265,6 +266,7 @@ void watchStartsFromStoredResourceVersionAndThenUsesInformerResourceVersion() { // ConfigReloadUtil calls the 'reloadProcedure' from below and we assert its // invocation ConfigurationUpdateStrategy strategy = new ConfigurationUpdateStrategy("strategy", () -> ++onEventCalls[0]); + List writtenResourceVersions = new ArrayList<>(); KubernetesNamespaceProvider namespaceProvider = mock(KubernetesNamespaceProvider.class); when(namespaceProvider.getNamespace()).thenReturn("default"); @@ -272,9 +274,11 @@ void watchStartsFromStoredResourceVersionAndThenUsesInformerResourceVersion() { KubernetesClientEventBasedConfigMapChangeDetector changeDetector = new KubernetesClientEventBasedConfigMapChangeDetector( coreV1Api, environment, ConfigReloadProperties.DEFAULT, strategy, locator, namespaceProvider, true); - changeDetector.start(Map.of("default", "17")); + changeDetector.start(Map.of("default", "17"), writtenResourceVersions::add); - Awaitilities.awaitUntil(10, 1000, () -> onEventCalls[0] == 1); + Awaitilities.awaitUntil(10, 1000, () -> onEventCalls[0] == 1 && writtenResourceVersions.size() == 1); + assertThat(writtenResourceVersions) + .containsExactly(new NamespaceAndResourceVersion("default", "43")); verify(getRequestedFor(urlMatching("^/api/v1/namespaces/default/configmaps.*")) .withQueryParam("watch", equalTo("false")) .withQueryParam("resourceVersion", equalTo("17"))); @@ -318,7 +322,7 @@ private void changeDetectorAssert(boolean haEnabled) { if (haEnabled) { assertThat(onEventCalls[0]).isZero(); - changeDetector.start(Map.of()); + changeDetector.start(Map.of(), null); } // all 4 events are caught diff --git a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java index 4d6a5be9b0..2a5af8074a 100644 --- a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java +++ b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java @@ -17,6 +17,7 @@ package org.springframework.cloud.kubernetes.client.config.reload; import java.time.Duration; +import java.util.ArrayList; import java.util.Base64; import java.util.List; import java.util.Map; @@ -261,6 +262,7 @@ void watchStartsFromStoredResourceVersionAndThenUsesInformerResourceVersion() { // ConfigReloadUtil calls the 'reloadProcedure' from below and we assert its // invocation ConfigurationUpdateStrategy strategy = new ConfigurationUpdateStrategy("strategy", () -> ++onEventCalls[0]); + List writtenResourceVersions = new ArrayList<>(); ConfigReloadProperties properties = new ConfigReloadProperties(false, false, true, ConfigReloadProperties.ReloadStrategy.REFRESH, ConfigReloadProperties.ReloadDetectionMode.EVENT, @@ -271,9 +273,11 @@ void watchStartsFromStoredResourceVersionAndThenUsesInformerResourceVersion() { KubernetesClientEventBasedSecretsChangeDetector changeDetector = new KubernetesClientEventBasedSecretsChangeDetector( coreV1Api, environment, properties, strategy, locator, namespaceProvider, true); - changeDetector.start(Map.of("default", "17")); + changeDetector.start(Map.of("default", "17"), writtenResourceVersions::add); - Awaitilities.awaitUntil(10, 1000, () -> onEventCalls[0] == 1); + Awaitilities.awaitUntil(10, 1000, () -> onEventCalls[0] == 1 && writtenResourceVersions.size() == 1); + Assertions.assertThat(writtenResourceVersions) + .containsExactly(new NamespaceAndResourceVersion("default", "43")); verify(getRequestedFor(urlMatching("/api/v1/namespaces/default/secrets.*")) .withQueryParam("watch", equalTo("false")) .withQueryParam("resourceVersion", equalTo("17"))); @@ -435,7 +439,7 @@ private void changeDetectorAssert(boolean haEnabled) { if (haEnabled) { Assertions.assertThat(onEventCalls[0]).isZero(); - changeDetector.start(Map.of()); + changeDetector.start(Map.of(), null); } // all 4 events are caught diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java index af599515d1..a5018dc5bd 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java @@ -21,6 +21,7 @@ import org.springframework.beans.factory.ObjectProvider; import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedConfigMapChangeDetector; import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedSecretsChangeDetector; +import org.springframework.cloud.kubernetes.client.config.reload.NamespaceAndResourceVersion; import org.springframework.cloud.kubernetes.commons.leader.election.events.StartLeadingEvent; import org.springframework.cloud.kubernetes.commons.leader.election.events.StopLeadingEvent; import org.springframework.context.event.EventListener; @@ -59,8 +60,18 @@ void onStartLeading(StartLeadingEvent event) { LOG.info(() -> "configuration watcher with identity : " + event.candidateIdentity() + " became leader at : " + Instant.ofEpochMilli(event.getTimestamp())); ConfigurationWatcherState state = stateStore.readOrCreate(); - configMapDetector.ifAvailable(detector -> detector.start(state.configMapResourceVersions())); - secretsDetector.ifAvailable(detector -> detector.start(state.secretResourceVersions())); + configMapDetector.ifAvailable(detector -> detector.start(state.configMapResourceVersions(), + this::writeConfigMapResourceVersion)); + secretsDetector.ifAvailable(detector -> detector.start(state.secretResourceVersions(), + this::writeSecretResourceVersion)); + } + + private void writeConfigMapResourceVersion(NamespaceAndResourceVersion resourceVersion) { + stateStore.writeConfigMapResourceVersion(resourceVersion.namespace(), resourceVersion.resourceVersion()); + } + + private void writeSecretResourceVersion(NamespaceAndResourceVersion resourceVersion) { + stateStore.writeSecretResourceVersion(resourceVersion.namespace(), resourceVersion.resourceVersion()); } @EventListener diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java index aa09234443..2f404026a5 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java @@ -27,6 +27,8 @@ import org.springframework.cloud.kubernetes.commons.leader.election.events.StopLeadingEvent; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -59,8 +61,8 @@ void onStartLeadingStartsBothDetectors() { coordinator.onStartLeading(new StartLeadingEvent("candidate")); - verify(configMapDetector).start(Map.of("default", "config-map-rv")); - verify(secretsDetector).start(Map.of("default", "secret-rv")); + verify(configMapDetector).start(eq(Map.of("default", "config-map-rv")), any()); + verify(secretsDetector).start(eq(Map.of("default", "secret-rv")), any()); } @Test From 51dd0e4e5f525f038b983531a84f4f456d5be323 Mon Sep 17 00:00:00 2001 From: wind57 Date: Wed, 19 Aug 2026 19:01:11 +0300 Subject: [PATCH 10/24] fix checkstyle Signed-off-by: wind57 --- .../reload/ConfigMapResourceEventHandler.java | 7 +++---- ...ClientEventBasedSecretsChangeDetector.java | 3 +-- .../reload/SecretResourceEventHandler.java | 19 +++++++++---------- ...ventBasedConfigMapChangeDetectorTests.java | 3 +-- ...tEventBasedSecretsChangeDetectorTests.java | 2 +- .../ha/ConfigurationWatcherHACoordinator.java | 8 ++++---- 6 files changed, 19 insertions(+), 23 deletions(-) diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/ConfigMapResourceEventHandler.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/ConfigMapResourceEventHandler.java index 746771f228..c6d1056dd4 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/ConfigMapResourceEventHandler.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/ConfigMapResourceEventHandler.java @@ -23,7 +23,6 @@ import io.kubernetes.client.openapi.models.V1ConfigMap; import io.kubernetes.client.openapi.models.V1ObjectMeta; import jakarta.annotation.Nullable; -import org.apache.commons.logging.LogFactory; import org.springframework.core.log.LogAccessor; @@ -32,7 +31,7 @@ */ final class ConfigMapResourceEventHandler implements ResourceEventHandler { - private static final LogAccessor LOG = new LogAccessor(LogFactory.getLog(ConfigMapResourceEventHandler.class)); + private static final LogAccessor LOG = new LogAccessor(ConfigMapResourceEventHandler.class); private final Consumer onEvent; @@ -77,8 +76,8 @@ public void onDelete(V1ConfigMap configMap, boolean deletedFinalStateUnknown) { private void writeResourceVersion(V1ConfigMap configMap) { if (resourceVersionWriter != null) { V1ObjectMeta metadata = configMap.getMetadata(); - resourceVersionWriter.accept(new NamespaceAndResourceVersion( - metadata.getNamespace(), metadata.getResourceVersion())); + resourceVersionWriter + .accept(new NamespaceAndResourceVersion(metadata.getNamespace(), metadata.getResourceVersion())); } } diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java index c45efa3e2b..ae85a8ba28 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java @@ -143,8 +143,7 @@ public final void start(Map storedResourceVersions, namespaces.forEach(namespace -> { SharedIndexInformer informer; - SecretResourceEventHandler handler = new SecretResourceEventHandler(LOG, this::onEvent, - resourceVersionWriter); + SecretResourceEventHandler handler = new SecretResourceEventHandler(this::onEvent, resourceVersionWriter); SharedInformerFactory factory = new SharedInformerFactory(apiClient); factories.add(factory); informer = factory diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/SecretResourceEventHandler.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/SecretResourceEventHandler.java index 71f8f43789..b0a12bb646 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/SecretResourceEventHandler.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/SecretResourceEventHandler.java @@ -21,29 +21,28 @@ import io.kubernetes.client.informer.ResourceEventHandler; import io.kubernetes.client.openapi.models.V1ObjectMeta; import io.kubernetes.client.openapi.models.V1Secret; - import jakarta.annotation.Nullable; + import org.springframework.core.log.LogAccessor; final class SecretResourceEventHandler implements ResourceEventHandler { - private final LogAccessor log; + private static final LogAccessor LOG = new LogAccessor(SecretResourceEventHandler.class); private final Consumer onEvent; @Nullable private final Consumer resourceVersionWriter; - SecretResourceEventHandler(LogAccessor log, Consumer onEvent, + SecretResourceEventHandler(Consumer onEvent, @Nullable Consumer resourceVersionWriter) { - this.log = log; this.onEvent = onEvent; this.resourceVersionWriter = resourceVersionWriter; } @Override public void onAdd(V1Secret secret) { - log.debug(() -> "Secret " + secret.getMetadata().getName() + " was added in namespace " + LOG.debug(() -> "Secret " + secret.getMetadata().getName() + " was added in namespace " + secret.getMetadata().getNamespace()); onEvent.accept(secret); writeResourceVersion(secret); @@ -51,11 +50,11 @@ public void onAdd(V1Secret secret) { @Override public void onUpdate(V1Secret oldSecret, V1Secret newSecret) { - log.debug(() -> "Secret " + newSecret.getMetadata().getName() + " was updated in namespace " + LOG.debug(() -> "Secret " + newSecret.getMetadata().getName() + " was updated in namespace " + newSecret.getMetadata().getNamespace()); if (KubernetesClientEventBasedSecretsChangeDetector.equals(oldSecret.getData(), newSecret.getData())) { - log.debug(() -> "data in secret has not changed, will not reload"); + LOG.debug(() -> "data in secret has not changed, will not reload"); } else { onEvent.accept(newSecret); @@ -65,7 +64,7 @@ public void onUpdate(V1Secret oldSecret, V1Secret newSecret) { @Override public void onDelete(V1Secret secret, boolean deletedFinalStateUnknown) { - log.debug(() -> "Secret " + secret.getMetadata().getName() + " was deleted in namespace " + LOG.debug(() -> "Secret " + secret.getMetadata().getName() + " was deleted in namespace " + secret.getMetadata().getNamespace()); onEvent.accept(secret); writeResourceVersion(secret); @@ -74,8 +73,8 @@ public void onDelete(V1Secret secret, boolean deletedFinalStateUnknown) { private void writeResourceVersion(V1Secret secret) { if (resourceVersionWriter != null) { V1ObjectMeta metadata = secret.getMetadata(); - resourceVersionWriter.accept(new NamespaceAndResourceVersion( - metadata.getNamespace(), metadata.getResourceVersion())); + resourceVersionWriter + .accept(new NamespaceAndResourceVersion(metadata.getNamespace(), metadata.getResourceVersion())); } } diff --git a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java index 29e3f6ef76..16c59f9270 100644 --- a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java +++ b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetectorTests.java @@ -277,8 +277,7 @@ void watchStartsFromStoredResourceVersionAndThenUsesInformerResourceVersion() { changeDetector.start(Map.of("default", "17"), writtenResourceVersions::add); Awaitilities.awaitUntil(10, 1000, () -> onEventCalls[0] == 1 && writtenResourceVersions.size() == 1); - assertThat(writtenResourceVersions) - .containsExactly(new NamespaceAndResourceVersion("default", "43")); + assertThat(writtenResourceVersions).containsExactly(new NamespaceAndResourceVersion("default", "43")); verify(getRequestedFor(urlMatching("^/api/v1/namespaces/default/configmaps.*")) .withQueryParam("watch", equalTo("false")) .withQueryParam("resourceVersion", equalTo("17"))); diff --git a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java index 2a5af8074a..3d6c5ea2e8 100644 --- a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java +++ b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java @@ -277,7 +277,7 @@ void watchStartsFromStoredResourceVersionAndThenUsesInformerResourceVersion() { Awaitilities.awaitUntil(10, 1000, () -> onEventCalls[0] == 1 && writtenResourceVersions.size() == 1); Assertions.assertThat(writtenResourceVersions) - .containsExactly(new NamespaceAndResourceVersion("default", "43")); + .containsExactly(new NamespaceAndResourceVersion("default", "43")); verify(getRequestedFor(urlMatching("/api/v1/namespaces/default/secrets.*")) .withQueryParam("watch", equalTo("false")) .withQueryParam("resourceVersion", equalTo("17"))); diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java index a5018dc5bd..7cad8b9398 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java @@ -60,10 +60,10 @@ void onStartLeading(StartLeadingEvent event) { LOG.info(() -> "configuration watcher with identity : " + event.candidateIdentity() + " became leader at : " + Instant.ofEpochMilli(event.getTimestamp())); ConfigurationWatcherState state = stateStore.readOrCreate(); - configMapDetector.ifAvailable(detector -> detector.start(state.configMapResourceVersions(), - this::writeConfigMapResourceVersion)); - secretsDetector.ifAvailable(detector -> detector.start(state.secretResourceVersions(), - this::writeSecretResourceVersion)); + configMapDetector.ifAvailable( + detector -> detector.start(state.configMapResourceVersions(), this::writeConfigMapResourceVersion)); + secretsDetector + .ifAvailable(detector -> detector.start(state.secretResourceVersions(), this::writeSecretResourceVersion)); } private void writeConfigMapResourceVersion(NamespaceAndResourceVersion resourceVersion) { From 63a6d004e88f7e8d83cce5d57b85c80bcb7ec933 Mon Sep 17 00:00:00 2001 From: wind57 Date: Wed, 19 Aug 2026 21:33:44 +0300 Subject: [PATCH 11/24] fix tests Signed-off-by: wind57 --- .../LeaseConfigurationWatcherStateStore.java | 2 +- ...rationWatcherHAAutoConfigurationTests.java | 11 +++--- ...onfigurationWatcherHACoordinatorTests.java | 34 +++++++++++++++++-- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java index aab19c9247..aa540b0b77 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java @@ -155,7 +155,7 @@ private static String serializeResourceVersions(Map resourceVers private static Map parseResourceVersions(String serializedResourceVersions) { if (!StringUtils.hasText(serializedResourceVersions)) { - return Map.of(); + return new HashMap<>(); // mutable on purpose } Map resourceVersions = new LinkedHashMap<>(); diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfigurationTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfigurationTests.java index 95d7758fbd..8e69ed31ba 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfigurationTests.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfigurationTests.java @@ -22,6 +22,7 @@ import org.springframework.boot.test.context.runner.ApplicationContextRunner; import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedConfigMapChangeDetector; import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedSecretsChangeDetector; +import org.springframework.cloud.kubernetes.configuration.watcher.ConfigurationWatcherConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -71,11 +72,6 @@ ApiClient apiClient() { return mock(ApiClient.class); } - @Bean - ConfigurationWatcherStateStore stateStore() { - return mock(ConfigurationWatcherStateStore.class); - } - @Bean KubernetesClientEventBasedConfigMapChangeDetector configMapDetector() { return mock(KubernetesClientEventBasedConfigMapChangeDetector.class); @@ -86,6 +82,11 @@ KubernetesClientEventBasedSecretsChangeDetector secretsDetector() { return mock(KubernetesClientEventBasedSecretsChangeDetector.class); } + @Bean + ConfigurationWatcherConfigurationProperties properties() { + return new ConfigurationWatcherConfigurationProperties(); + } + } } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java index 2f404026a5..7b8843e104 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinatorTests.java @@ -17,6 +17,7 @@ package org.springframework.cloud.kubernetes.configuration.watcher.ha; import java.util.Map; +import java.util.function.Consumer; import org.junit.jupiter.api.Test; @@ -29,6 +30,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -50,9 +52,22 @@ void onStartLeadingStartsBothDetectors() { ObjectProvider.class); when(configMapProvider.getIfAvailable()).thenReturn(configMapDetector); + doAnswer(invocation -> { + Consumer consumer = invocation.getArgument(0); + consumer.accept(configMapDetector); + return null; + }).when(configMapProvider).ifAvailable(any()); + ObjectProvider secretsProvider = mock(ObjectProvider.class); when(secretsProvider.getIfAvailable()).thenReturn(secretsDetector); - ConfigurationWatcherStateStore stateStore = mock(ConfigurationWatcherStateStore.class); + + doAnswer(invocation -> { + Consumer consumer = invocation.getArgument(0); + consumer.accept(secretsDetector); + return null; + }).when(secretsProvider).ifAvailable(any()); + + ConfigurationWatcherStateStore stateStore = mock(LeaseConfigurationWatcherStateStore.class); when(stateStore.readOrCreate()).thenReturn( new ConfigurationWatcherState(Map.of("default", "config-map-rv"), Map.of("default", "secret-rv"))); @@ -77,9 +92,22 @@ void onStopLeadingStopsBothDetectors() { ObjectProvider.class); when(configMapProvider.getIfAvailable()).thenReturn(configMapDetector); + doAnswer(invocation -> { + Consumer consumer = invocation.getArgument(0); + consumer.accept(configMapDetector); + return null; + }).when(configMapProvider).ifAvailable(any()); + ObjectProvider secretsProvider = mock(ObjectProvider.class); when(secretsProvider.getIfAvailable()).thenReturn(secretsDetector); - ConfigurationWatcherStateStore stateStore = mock(ConfigurationWatcherStateStore.class); + + doAnswer(invocation -> { + Consumer consumer = invocation.getArgument(0); + consumer.accept(secretsDetector); + return null; + }).when(secretsProvider).ifAvailable(any()); + + ConfigurationWatcherStateStore stateStore = mock(LeaseConfigurationWatcherStateStore.class); ConfigurationWatcherHACoordinator coordinator = new ConfigurationWatcherHACoordinator(configMapProvider, secretsProvider, stateStore); @@ -97,7 +125,7 @@ void failsWhenNeitherDetectorIsAvailable() { when(configMapProvider.getIfAvailable()).thenReturn(null); ObjectProvider secretsProvider = mock(ObjectProvider.class); when(secretsProvider.getIfAvailable()).thenReturn(null); - ConfigurationWatcherStateStore stateStore = mock(ConfigurationWatcherStateStore.class); + ConfigurationWatcherStateStore stateStore = mock(LeaseConfigurationWatcherStateStore.class); assertThatThrownBy(() -> new ConfigurationWatcherHACoordinator(configMapProvider, secretsProvider, stateStore)) .isInstanceOf(IllegalStateException.class) From e9476bfc2f63ecf3f5ec5f0a7ac3797dacb0e4ee Mon Sep 17 00:00:00 2001 From: wind57 Date: Fri, 21 Aug 2026 17:57:31 +0300 Subject: [PATCH 12/24] started adding ITs Signed-off-by: wind57 --- ...ientEventBasedConfigMapChangeDetector.java | 5 ++ ...ClientEventBasedSecretsChangeDetector.java | 5 ++ .../pom.xml | 4 ++ ...nfigurationWatcherHAAutoConfiguration.java | 6 ++ .../ha/ConfigurationWatcherHACoordinator.java | 43 +++++++----- ...ernetesClientConfigurationWatcherHaIT.java | 65 +++++++++++++++++++ .../k3s/NativeClientIntegrationTest.java | 5 ++ .../NativeClientIntegrationTestExtension.java | 5 +- .../NativeClientKubernetesFixture.java | 7 +- 9 files changed, 126 insertions(+), 19 deletions(-) create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/KubernetesClientConfigurationWatcherHaIT.java diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java index 41dd7c82af..2d4f1dc965 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java @@ -109,8 +109,13 @@ void inform() { // In HA mode, defer informer startup until this instance acquires leadership. // The leader callback restores the persisted state and then starts the informers. if (!haEnabled) { + LOG.info(() -> "config watcher HA is disabled : starting configmap informers immediately"); start(Map.of(), null); } + else { + LOG.info(() -> "config watcher HA is enabled : deferring configmap informer startup " + + "until leadership is acquired"); + } } public final void start(Map storedResourceVersions, diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java index ae85a8ba28..21b0da28e9 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java @@ -113,8 +113,13 @@ void inform() { // In HA mode, defer informer startup until this instance acquires leadership. // The leader callback restores the persisted state and then starts the informers. if (!haEnabled) { + LOG.info(() -> "config watcher HA is disabled : starting configmap informers immediately"); start(Map.of(), null); } + else { + LOG.info(() -> "config watcher HA is enabled : deferring configmap informer startup " + + "until leadership is acquired"); + } } public final void start(Map storedResourceVersions, diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/pom.xml b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/pom.xml index d311d6f6b6..17fc5cf973 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/pom.xml +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/pom.xml @@ -22,6 +22,10 @@ org.springframework.cloud spring-cloud-starter-kubernetes-client-all + + org.springframework.cloud + spring-cloud-kubernetes-client-leader + org.springframework.cloud spring-cloud-starter-bus-amqp diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfiguration.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfiguration.java index 16447f1343..812f240781 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfiguration.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHAAutoConfiguration.java @@ -20,12 +20,16 @@ import io.kubernetes.client.openapi.apis.CoordinationV1Api; import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnCloudPlatform; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.cloud.CloudPlatform; +import org.springframework.cloud.kubernetes.client.KubernetesClientAutoConfiguration; import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedConfigMapChangeDetector; import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedSecretsChangeDetector; +import org.springframework.cloud.kubernetes.client.leader.election.KubernetesClientLeaderElectionCallbacksAutoConfiguration; import org.springframework.cloud.kubernetes.commons.leader.election.ConditionalOnLeaderElectionEnabled; import org.springframework.cloud.kubernetes.configuration.watcher.ConfigurationWatcherConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -45,6 +49,8 @@ @ConditionalOnConfigurationWatcherHAEnabled @ConditionalOnLeaderElectionEnabled @ConditionalOnBean(ApiClient.class) +@AutoConfigureAfter(KubernetesClientAutoConfiguration.class) +@AutoConfigureBefore(KubernetesClientLeaderElectionCallbacksAutoConfiguration.class) class ConfigurationWatcherHAAutoConfiguration { @Bean diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java index 7cad8b9398..bf8d50fda4 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/ConfigurationWatcherHACoordinator.java @@ -18,13 +18,16 @@ import java.time.Instant; +import org.jspecify.annotations.NonNull; + import org.springframework.beans.factory.ObjectProvider; import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedConfigMapChangeDetector; import org.springframework.cloud.kubernetes.client.config.reload.KubernetesClientEventBasedSecretsChangeDetector; import org.springframework.cloud.kubernetes.client.config.reload.NamespaceAndResourceVersion; import org.springframework.cloud.kubernetes.commons.leader.election.events.StartLeadingEvent; import org.springframework.cloud.kubernetes.commons.leader.election.events.StopLeadingEvent; -import org.springframework.context.event.EventListener; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.ApplicationListener; import org.springframework.core.log.LogAccessor; /** @@ -32,19 +35,19 @@ * * @author wind57 */ -public final class ConfigurationWatcherHACoordinator { +final class ConfigurationWatcherHACoordinator implements ApplicationListener<@NonNull ApplicationEvent> { private static final LogAccessor LOG = new LogAccessor(ConfigurationWatcherHACoordinator.class); - private final ObjectProvider configMapDetector; + private final ObjectProvider<@NonNull KubernetesClientEventBasedConfigMapChangeDetector> configMapDetector; - private final ObjectProvider secretsDetector; + private final ObjectProvider<@NonNull KubernetesClientEventBasedSecretsChangeDetector> secretsDetector; private final ConfigurationWatcherStateStore stateStore; - public ConfigurationWatcherHACoordinator( - ObjectProvider configMapDetector, - ObjectProvider secretsDetector, + ConfigurationWatcherHACoordinator( + ObjectProvider<@NonNull KubernetesClientEventBasedConfigMapChangeDetector> configMapDetector, + ObjectProvider<@NonNull KubernetesClientEventBasedSecretsChangeDetector> secretsDetector, ConfigurationWatcherStateStore stateStore) { if (configMapDetector.getIfAvailable() == null && secretsDetector.getIfAvailable() == null) { throw new IllegalStateException( @@ -55,7 +58,16 @@ public ConfigurationWatcherHACoordinator( this.stateStore = stateStore; } - @EventListener + @Override + public void onApplicationEvent(ApplicationEvent event) { + if (event instanceof StartLeadingEvent startLeadingEvent) { + onStartLeading(startLeadingEvent); + } + else if (event instanceof StopLeadingEvent stopLeadingEvent) { + onStopLeading(stopLeadingEvent); + } + } + void onStartLeading(StartLeadingEvent event) { LOG.info(() -> "configuration watcher with identity : " + event.candidateIdentity() + " became leader at : " + Instant.ofEpochMilli(event.getTimestamp())); @@ -66,6 +78,13 @@ void onStartLeading(StartLeadingEvent event) { .ifAvailable(detector -> detector.start(state.secretResourceVersions(), this::writeSecretResourceVersion)); } + void onStopLeading(StopLeadingEvent event) { + LOG.info(() -> "configuration watcher with identity : " + event.candidateIdentity() + + " stopped being a leader at : " + Instant.ofEpochMilli(event.getTimestamp())); + secretsDetector.ifAvailable(KubernetesClientEventBasedSecretsChangeDetector::stop); + configMapDetector.ifAvailable(KubernetesClientEventBasedConfigMapChangeDetector::stop); + } + private void writeConfigMapResourceVersion(NamespaceAndResourceVersion resourceVersion) { stateStore.writeConfigMapResourceVersion(resourceVersion.namespace(), resourceVersion.resourceVersion()); } @@ -74,12 +93,4 @@ private void writeSecretResourceVersion(NamespaceAndResourceVersion resourceVers stateStore.writeSecretResourceVersion(resourceVersion.namespace(), resourceVersion.resourceVersion()); } - @EventListener - void onStopLeading(StopLeadingEvent event) { - LOG.info(() -> "configuration watcher with identity : " + event.candidateIdentity() - + " stopped being a leader at : " + Instant.ofEpochMilli(event.getTimestamp())); - secretsDetector.ifAvailable(KubernetesClientEventBasedSecretsChangeDetector::stop); - configMapDetector.ifAvailable(KubernetesClientEventBasedConfigMapChangeDetector::stop); - } - } diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/KubernetesClientConfigurationWatcherHaIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/KubernetesClientConfigurationWatcherHaIT.java new file mode 100644 index 0000000000..3002d81aa9 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/KubernetesClientConfigurationWatcherHaIT.java @@ -0,0 +1,65 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher.ha; + +import org.junit.jupiter.api.Test; +import org.testcontainers.k3s.K3sContainer; + +import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; +import org.springframework.cloud.kubernetes.integration.tests.commons.k3s.NativeClientIntegrationTest; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Verifies the initial Kubernetes-client configuration watcher HA wiring. + * + * @author wind57 + */ +@NativeClientIntegrationTest(withImages = { "spring-cloud-kubernetes-configuration-watcher" }, + rbacNamespaces = "default", + configurationWatcher = @NativeClientIntegrationTest.ConfigurationWatcher(enabled = true, enableHa = true, + refreshDelay = "0", reloadEnabled = false)) +class KubernetesClientConfigurationWatcherHaIT { + + private static final String CONFIGURATION_WATCHER_APP = "spring-cloud-kubernetes-configuration-watcher"; + + private static final String LEADER_ELECTION_LEASE = "spring-k8s-leader-election-lock"; + + private static final String CONFIGURATION_WATCHER_STATE_LEASE = "configuration-watcher-ha"; + + /** + * 

+	 *     - start one configuration watcher with HA enabled
+	 *     - wait until it becomes leader and creates the HA state lease
+	 *     - verify that both the leader-election and state leases exist
+	 *
+ */ + @Test + void startsAsLeaderAndCreatesHaStateLease(K3sContainer container) throws Exception { + Commons.waitForLogStatement("Creating watcher HA lease with name : " + CONFIGURATION_WATCHER_STATE_LEASE, + container, CONFIGURATION_WATCHER_APP); + + String leases = container + .execInContainer("kubectl", "get", "lease", LEADER_ELECTION_LEASE, CONFIGURATION_WATCHER_STATE_LEASE, + "--namespace", "default", "--output", "name") + .getStdout(); + + assertThat(leases).contains("lease.coordination.k8s.io/" + LEADER_ELECTION_LEASE, + "lease.coordination.k8s.io/" + CONFIGURATION_WATCHER_STATE_LEASE); + } + +} diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTest.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTest.java index ca86ef3c56..175f897e87 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTest.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTest.java @@ -107,6 +107,11 @@ */ boolean reloadEnabled() default true; + /** + * enable configuration watcher HA. + */ + boolean enableHa() default false; + /** * what namespaces to be watched. 'SPRING_CLOUD_KUBERNETES_RELOAD_NAMESPACES_0' * and so on diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java index 0195986c10..942dc9ab15 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java @@ -113,7 +113,8 @@ public void beforeAll(ExtensionContext context) throws Exception { if (scenario.configurationWatcher().enabled()) { nativeClientKubernetesFixture.configWatcher(Phase.CREATE, scenario.configurationWatcher().refreshDelay(), scenario.configurationWatcher().reloadEnabled(), scenario.configurationWatcher().watchNamespaces(), - scenario.configurationWatcher().kafkaEnabled(), scenario.configurationWatcher().rabbitMqEnabled()); + scenario.configurationWatcher().kafkaEnabled(), scenario.configurationWatcher().rabbitMqEnabled(), + scenario.configurationWatcher().enableHa()); } // 11. deploy discovery server @@ -157,7 +158,7 @@ public void afterAll(ExtensionContext context) throws Exception { // 5. delete configuration watcher. if (scenario.configurationWatcher().enabled()) { - nativeClientKubernetesFixture.configWatcher(Phase.DELETE, "", false, null, false, false); + nativeClientKubernetesFixture.configWatcher(Phase.DELETE, "", false, null, false, false, false); } // 6. delete all namespaces diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java index 4d5ed858c1..02fde35f02 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java @@ -507,7 +507,7 @@ public void externalName(Phase phase) { } public void configWatcher(Phase phase, String refreshDelay, boolean reloadEnabled, String[] watchNamespaces, - boolean kafkaEnabled, boolean rabbitMqEnabled) { + boolean kafkaEnabled, boolean rabbitMqEnabled, boolean enableHa) { V1Deployment deployment = yaml("config-watcher/deployment.yaml", V1Deployment.class); V1Service service = yaml("config-watcher/service.yaml", V1Service.class); @@ -522,6 +522,11 @@ public void configWatcher(Phase phase, String refreshDelay, boolean reloadEnable .value("DEBUG")); envVars.add(new V1EnvVar().name("SPRING_CLOUD_KUBERNETES_SECRETS_ENABLED").value("TRUE")); + if (enableHa) { + envVars.add(new V1EnvVar().name("SPRING_CLOUD_KUBERNETES_LEADER_ELECTION_ENABLED").value("true")); + envVars.add(new V1EnvVar().name("SPRING_CLOUD_KUBERNETES_CONFIGURATION_WATCHER_HA_ENABLED").value("true")); + } + if (kafkaEnabled) { envVars.add(new V1EnvVar().name("SPRING_PROFILES_ACTIVE").value("bus-kafka")); envVars.add(new V1EnvVar().name("SPRING_CLOUD_BUS_DESTINATION").value("app")); From bd78e77edc26827df77cd63b45a7271b20bd1fba Mon Sep 17 00:00:00 2001 From: wind57 Date: Sat, 22 Aug 2026 11:02:21 +0300 Subject: [PATCH 13/24] cleanup so far in the IP Signed-off-by: wind57 --- ...ernetesClientConfigurationWatcherHaIT.java | 69 +++++++++++++------ .../k3s/NativeClientIntegrationTest.java | 5 ++ .../NativeClientIntegrationTestExtension.java | 4 +- .../NativeClientKubernetesFixture.java | 11 +-- 4 files changed, 63 insertions(+), 26 deletions(-) diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/KubernetesClientConfigurationWatcherHaIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/KubernetesClientConfigurationWatcherHaIT.java index 3002d81aa9..64a9b677fb 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/KubernetesClientConfigurationWatcherHaIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/KubernetesClientConfigurationWatcherHaIT.java @@ -16,10 +16,13 @@ package org.springframework.cloud.kubernetes.configuration.watcher.ha; +import java.util.Arrays; +import java.util.List; + import org.junit.jupiter.api.Test; import org.testcontainers.k3s.K3sContainer; -import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; +import org.springframework.cloud.kubernetes.integration.tests.commons.Awaitilities; import org.springframework.cloud.kubernetes.integration.tests.commons.k3s.NativeClientIntegrationTest; import static org.assertj.core.api.Assertions.assertThat; @@ -32,34 +35,60 @@ @NativeClientIntegrationTest(withImages = { "spring-cloud-kubernetes-configuration-watcher" }, rbacNamespaces = "default", configurationWatcher = @NativeClientIntegrationTest.ConfigurationWatcher(enabled = true, enableHa = true, - refreshDelay = "0", reloadEnabled = false)) + replicas = 2, refreshDelay = "0", reloadEnabled = false)) class KubernetesClientConfigurationWatcherHaIT { - private static final String CONFIGURATION_WATCHER_APP = "spring-cloud-kubernetes-configuration-watcher"; - - private static final String LEADER_ELECTION_LEASE = "spring-k8s-leader-election-lock"; - - private static final String CONFIGURATION_WATCHER_STATE_LEASE = "configuration-watcher-ha"; - /** * 
-	 *     - start one configuration watcher with HA enabled
-	 *     - wait until it becomes leader and creates the HA state lease
-	 *     - verify that both the leader-election and state leases exist
+	 *     - start two configuration watcher replicas with HA enabled
+	 *     - wait until both replicas are running
+	 *     - verify that exactly one replica holds the leader-election lease
 	 *
*/ @Test - void startsAsLeaderAndCreatesHaStateLease(K3sContainer container) throws Exception { - Commons.waitForLogStatement("Creating watcher HA lease with name : " + CONFIGURATION_WATCHER_STATE_LEASE, - container, CONFIGURATION_WATCHER_APP); + void startsTwoReplicasWithSingleLeader(K3sContainer container) { + Awaitilities.awaitUntilAsserted(120, 1000, () -> { + try { + List runningPods = runningPods(container); + assertThat(runningPods).hasSize(2); + + // we have two replicas, one is the HA leader + String holderIdentity = container + .execInContainer("sh", "-c", + "kubectl get lease -n default spring-k8s-leader-election-lock" + + " -o jsonpath='{.spec.holderIdentity}'") + .getStdout() + .trim(); + + assertThat(holderIdentity).isNotBlank(); + assertThat(runningPods).contains(holderIdentity); + } + catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + /** + * get both pods as part of the replica of the deployment. + */ + private List runningPods(K3sContainer container) { + try { + String runningPods = container + .execInContainer("sh", "-c", + "kubectl get pods -n default -l app=spring-cloud-kubernetes-configuration-watcher" + + " --field-selector=status.phase=Running" + + " -o jsonpath='{.items[*].metadata.name}'") + .getStdout() + .trim(); + + return runningPods.isEmpty() ? List.of() + : Arrays.stream(runningPods.split("\\s+")).toList(); - String leases = container - .execInContainer("kubectl", "get", "lease", LEADER_ELECTION_LEASE, CONFIGURATION_WATCHER_STATE_LEASE, - "--namespace", "default", "--output", "name") - .getStdout(); + } catch (Exception e) { + throw new RuntimeException(e); + } - assertThat(leases).contains("lease.coordination.k8s.io/" + LEADER_ELECTION_LEASE, - "lease.coordination.k8s.io/" + CONFIGURATION_WATCHER_STATE_LEASE); } } diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTest.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTest.java index 175f897e87..aad00ecfab 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTest.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTest.java @@ -112,6 +112,11 @@ */ boolean enableHa() default false; + /** + * number of configuration watcher replicas. + */ + int replicas() default 1; + /** * what namespaces to be watched. 'SPRING_CLOUD_KUBERNETES_RELOAD_NAMESPACES_0' * and so on diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java index 942dc9ab15..9ef474aacf 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/k3s/NativeClientIntegrationTestExtension.java @@ -114,7 +114,7 @@ public void beforeAll(ExtensionContext context) throws Exception { nativeClientKubernetesFixture.configWatcher(Phase.CREATE, scenario.configurationWatcher().refreshDelay(), scenario.configurationWatcher().reloadEnabled(), scenario.configurationWatcher().watchNamespaces(), scenario.configurationWatcher().kafkaEnabled(), scenario.configurationWatcher().rabbitMqEnabled(), - scenario.configurationWatcher().enableHa()); + scenario.configurationWatcher().enableHa(), scenario.configurationWatcher().replicas()); } // 11. deploy discovery server @@ -158,7 +158,7 @@ public void afterAll(ExtensionContext context) throws Exception { // 5. delete configuration watcher. if (scenario.configurationWatcher().enabled()) { - nativeClientKubernetesFixture.configWatcher(Phase.DELETE, "", false, null, false, false, false); + nativeClientKubernetesFixture.configWatcher(Phase.DELETE, "", false, null, false, false, false, 1); } // 6. delete all namespaces diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java index 02fde35f02..6f25e4b435 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/NativeClientKubernetesFixture.java @@ -507,10 +507,11 @@ public void externalName(Phase phase) { } public void configWatcher(Phase phase, String refreshDelay, boolean reloadEnabled, String[] watchNamespaces, - boolean kafkaEnabled, boolean rabbitMqEnabled, boolean enableHa) { + boolean kafkaEnabled, boolean rabbitMqEnabled, boolean enableHa, int replicas) { V1Deployment deployment = yaml("config-watcher/deployment.yaml", V1Deployment.class); V1Service service = yaml("config-watcher/service.yaml", V1Service.class); + deployment.getSpec().setReplicas(replicas); List envVars = new ArrayList<>(); envVars @@ -587,9 +588,10 @@ private String secretName(V1Secret secret) { private void waitForDeployment(String namespace, V1Deployment deployment) { String deploymentName = deploymentName(deployment); + int expectedReplicas = deployment.getSpec().getReplicas() == null ? 1 : deployment.getSpec().getReplicas(); Awaitilities.awaitUntil(600, 1000, () -> { try { - return isDeploymentReady(deploymentName, namespace); + return isDeploymentReady(deploymentName, namespace, expectedReplicas); } catch (ApiException e) { throw new RuntimeException(e); @@ -726,7 +728,8 @@ private void waitForDeploymentPodsToBeDeleted(Map labels, String } - private boolean isDeploymentReady(String deploymentName, String namespace) throws ApiException { + private boolean isDeploymentReady(String deploymentName, String namespace, int expectedReplicas) + throws ApiException { V1DeploymentList deployments = appsV1Api.listNamespacedDeployment(namespace) .fieldSelector("metadata.name=" + deploymentName) .execute(); @@ -739,7 +742,7 @@ private boolean isDeploymentReady(String deploymentName, String namespace) throw logDeploymentConditions(deployment.getStatus().getConditions(), deployment.getMetadata().getNamespace()); LOG.info("Available replicas for " + deploymentName + ": " + (availableReplicas == null ? 0 : availableReplicas)); - return availableReplicas != null && availableReplicas >= 1; + return availableReplicas != null && availableReplicas >= expectedReplicas; } else { return false; From b21d0dfe7d40524297f1371aa2e00ae083280763 Mon Sep 17 00:00:00 2001 From: wind57 Date: Sun, 23 Aug 2026 14:51:06 +0300 Subject: [PATCH 14/24] more changes added to the test Signed-off-by: wind57 --- ...ernetesClientConfigurationWatcherHaIT.java | 203 ++++++++++++++++++ ...ernetesClientConfigurationWatcherHaIT.java | 94 -------- 2 files changed, 203 insertions(+), 94 deletions(-) create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java delete mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/KubernetesClientConfigurationWatcherHaIT.java diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java new file mode 100644 index 0000000000..a14c2f4a2a --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java @@ -0,0 +1,203 @@ +/* + * Copyright 2013-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher; + +import java.util.Arrays; +import java.util.List; + +import com.github.tomakehurst.wiremock.client.WireMock; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.k3s.K3sContainer; + +import org.springframework.cloud.kubernetes.integration.tests.commons.Awaitilities; +import org.springframework.cloud.kubernetes.integration.tests.commons.k3s.NativeClientIntegrationTest; +import org.springframework.cloud.kubernetes.integration.tests.commons.native_client.NativeClientKubernetesFixture; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Verifies the initial Kubernetes-client configuration watcher HA wiring. + * + * @author wind57 + */ +@NativeClientIntegrationTest(withImages = { "spring-cloud-kubernetes-configuration-watcher" }, + wiremock = @NativeClientIntegrationTest.Wiremock(enabled = true, namespaces = "default", withNodePort = true), + rbacNamespaces = "default", + configurationWatcher = @NativeClientIntegrationTest.ConfigurationWatcher(enabled = true, enableHa = true, + replicas = 2, refreshDelay = "0", reloadEnabled = false)) +class KubernetesClientConfigurationWatcherHaIT { + + @BeforeAll + static void beforeAll(NativeClientKubernetesFixture fixture) { + TestUtil.configureWireMock(); + TestUtil.createConfigMap(fixture, "default"); + } + + @AfterAll + static void afterAll(NativeClientKubernetesFixture fixture) { + TestUtil.deleteConfigMap(fixture, "default"); + } + + /** + * 
+	 *     - start two configuration watcher replicas with HA enabled
+	 *     - wait until both replicas are running
+	 *     - verify that exactly one replica holds the leader-election lease
+	 *     - wait until the initial ConfigMap resource version is stored in the HA Lease
+	 *     - update the ConfigMap once
+	 *     - verify that the change triggers exactly one actuator refresh
+	 *     - verify that the updated resource version is stored in the HA Lease
+	 *
+ */ + @Test + void persistsResourceVersionAndTriggersRefreshWithTwoReplicas(K3sContainer container) { + + // 1. we have two replicas running + // 2. only one is the HA leader + Awaitilities.awaitUntilAsserted(120, 1000, () -> { + try { + List runningPods = runningPods(container); + assertThat(runningPods).hasSize(2); + + // we have two replicas, one is the HA leader + String exec = """ + kubectl get lease -n default spring-k8s-leader-election-lock \\ + -o "jsonpath={.spec.holderIdentity}" + """; + + String holderIdentity = container + .execInContainer("sh", "-c", exec) + .getStdout() + .trim(); + + assertThat(holderIdentity).isNotBlank(); + assertThat(runningPods).contains(holderIdentity); + } + catch (Exception e) { + throw new RuntimeException(e); + } + }); + + // 3. resource version of the configmap is present in out store + String initialResourceVersion = configMapResourceVersion(container); + Awaitilities.awaitUntilAsserted(120, 1000, + () -> assertThat(configMapResourceVersionInStateLease(container)) + .isEqualTo(initialResourceVersion)); + + // 4. once we update the configmap, resourceVersion changes, and we have it in out store. + WireMock.resetAllRequests(); + patchConfigMap(container); + String updatedResourceVersion = configMapResourceVersion(container); + + // 5. because of the update in the configmap, watcher caught that and sent a + // refresh call to the actuator ( wiremock in our test ) + TestUtil.verifyActuatorCalled(1); + + // 6. the new resourceVersion is not equal to the previous one + // 7. we have the lastest resourceVersion in our store + Awaitilities.awaitUntilAsserted(120, 1000, + () -> { + String afterPatchResourceVersion = configMapResourceVersionInStateLease(container); + assertThat(afterPatchResourceVersion).isNotEqualTo(initialResourceVersion); + assertThat(afterPatchResourceVersion).isEqualTo(updatedResourceVersion); + }); + } + + /** + * get both pods as part of the replica of the deployment. + */ + private List runningPods(K3sContainer container) { + + String exec = """ + kubectl get pods -n default -l app=spring-cloud-kubernetes-configuration-watcher \\ + --field-selector=status.phase=Running \\ + -o "jsonpath={.items[*].metadata.name}" + """; + + try { + String runningPods = container + .execInContainer("sh", "-c", exec) + .getStdout() + .trim(); + + return runningPods.isEmpty() ? List.of() + : Arrays.stream(runningPods.split("\\s+")).toList(); + + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * return the resourceVersion from the 'service-wiremock' configmap. + */ + private String configMapResourceVersion(K3sContainer container) { + + String exec = """ + kubectl get configmap service-wiremock -n default \\ + -o "jsonpath={.metadata.resourceVersion}" + """; + + try { + return container + .execInContainer("sh", "-c", exec) + .getStdout() + .trim(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + private String configMapResourceVersionInStateLease(K3sContainer container) { + + String exec = """ + kubectl get lease -n default configuration-watcher-ha \\ + -o "jsonpath={.metadata.annotations['spring\\.cloud\\.kubernetes\\.configuration\\.watcher/configmap-resource-version']}" + """; + + try { + // default=123 + String storedResourceVersion = container + .execInContainer("sh", "-c", exec) + .getStdout() + .trim(); + // get only the 123 part + return storedResourceVersion.substring("default=".length()); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void patchConfigMap(K3sContainer container) { + String exec = """ + kubectl patch configmap service-wiremock -n default --type merge \\ + -p '{"data":{"foo":"updated"}}' + """; + + try { + container.execInContainer("sh", "-c", exec); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/KubernetesClientConfigurationWatcherHaIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/KubernetesClientConfigurationWatcherHaIT.java deleted file mode 100644 index 64a9b677fb..0000000000 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/KubernetesClientConfigurationWatcherHaIT.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2013-present the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.kubernetes.configuration.watcher.ha; - -import java.util.Arrays; -import java.util.List; - -import org.junit.jupiter.api.Test; -import org.testcontainers.k3s.K3sContainer; - -import org.springframework.cloud.kubernetes.integration.tests.commons.Awaitilities; -import org.springframework.cloud.kubernetes.integration.tests.commons.k3s.NativeClientIntegrationTest; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * Verifies the initial Kubernetes-client configuration watcher HA wiring. - * - * @author wind57 - */ -@NativeClientIntegrationTest(withImages = { "spring-cloud-kubernetes-configuration-watcher" }, - rbacNamespaces = "default", - configurationWatcher = @NativeClientIntegrationTest.ConfigurationWatcher(enabled = true, enableHa = true, - replicas = 2, refreshDelay = "0", reloadEnabled = false)) -class KubernetesClientConfigurationWatcherHaIT { - - /** - * 
-	 *     - start two configuration watcher replicas with HA enabled
-	 *     - wait until both replicas are running
-	 *     - verify that exactly one replica holds the leader-election lease
-	 *
- */ - @Test - void startsTwoReplicasWithSingleLeader(K3sContainer container) { - Awaitilities.awaitUntilAsserted(120, 1000, () -> { - try { - List runningPods = runningPods(container); - assertThat(runningPods).hasSize(2); - - // we have two replicas, one is the HA leader - String holderIdentity = container - .execInContainer("sh", "-c", - "kubectl get lease -n default spring-k8s-leader-election-lock" - + " -o jsonpath='{.spec.holderIdentity}'") - .getStdout() - .trim(); - - assertThat(holderIdentity).isNotBlank(); - assertThat(runningPods).contains(holderIdentity); - } - catch (Exception e) { - throw new RuntimeException(e); - } - }); - } - - /** - * get both pods as part of the replica of the deployment. - */ - private List runningPods(K3sContainer container) { - try { - String runningPods = container - .execInContainer("sh", "-c", - "kubectl get pods -n default -l app=spring-cloud-kubernetes-configuration-watcher" - + " --field-selector=status.phase=Running" - + " -o jsonpath='{.items[*].metadata.name}'") - .getStdout() - .trim(); - - return runningPods.isEmpty() ? List.of() - : Arrays.stream(runningPods.split("\\s+")).toList(); - - } catch (Exception e) { - throw new RuntimeException(e); - } - - } - -} From e2223206fdce286195ba6a22dbe3b0e514236543 Mon Sep 17 00:00:00 2001 From: wind57 Date: Sun, 23 Aug 2026 14:51:56 +0300 Subject: [PATCH 15/24] more changes added to the test Signed-off-by: wind57 --- .../watcher/KubernetesClientConfigurationWatcherHaIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java index a14c2f4a2a..209ab8e46c 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java @@ -110,7 +110,7 @@ void persistsResourceVersionAndTriggersRefreshWithTwoReplicas(K3sContainer conta TestUtil.verifyActuatorCalled(1); // 6. the new resourceVersion is not equal to the previous one - // 7. we have the lastest resourceVersion in our store + // 7. we have the latest resourceVersion in our store Awaitilities.awaitUntilAsserted(120, 1000, () -> { String afterPatchResourceVersion = configMapResourceVersionInStateLease(container); From bbd8213ae05e1388bec62a7e9d8f9f4315790d98 Mon Sep 17 00:00:00 2001 From: wind57 Date: Mon, 24 Aug 2026 23:58:23 +0300 Subject: [PATCH 16/24] more IT Signed-off-by: wind57 --- ...ernetesClientConfigurationWatcherHaIT.java | 148 ++++++++++++++---- 1 file changed, 116 insertions(+), 32 deletions(-) diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java index 209ab8e46c..f3a4fe7c6f 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java @@ -18,6 +18,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Optional; import com.github.tomakehurst.wiremock.client.WireMock; import org.junit.jupiter.api.AfterAll; @@ -36,10 +37,10 @@ * * @author wind57 */ -@NativeClientIntegrationTest(withImages = { "spring-cloud-kubernetes-configuration-watcher" }, - wiremock = @NativeClientIntegrationTest.Wiremock(enabled = true, namespaces = "default", withNodePort = true), - rbacNamespaces = "default", - configurationWatcher = @NativeClientIntegrationTest.ConfigurationWatcher(enabled = true, enableHa = true, + @NativeClientIntegrationTest(withImages = { "spring-cloud-kubernetes-configuration-watcher" }, + wiremock = @NativeClientIntegrationTest.Wiremock(enabled = true, namespaces = "default", withNodePort = true), + rbacNamespaces = "default", + configurationWatcher = @NativeClientIntegrationTest.ConfigurationWatcher(enabled = true, enableHa = true, replicas = 2, refreshDelay = "0", reloadEnabled = false)) class KubernetesClientConfigurationWatcherHaIT { @@ -63,29 +64,24 @@ static void afterAll(NativeClientKubernetesFixture fixture) { * - update the ConfigMap once * - verify that the change triggers exactly one actuator refresh * - verify that the updated resource version is stored in the HA Lease + * - delete the current leader + * - update the ConfigMap while no watcher is leading + * - verify that no actuator refresh is sent before leadership changes + * - verify that the new leader replays the missed event * */ @Test - void persistsResourceVersionAndTriggersRefreshWithTwoReplicas(K3sContainer container) { + void persistsResourceVersionAndReplaysChangeAfterLeaderLoss(K3sContainer container) { // 1. we have two replicas running // 2. only one is the HA leader Awaitilities.awaitUntilAsserted(120, 1000, () -> { try { + // we have two replicas, one is the HA leader List runningPods = runningPods(container); assertThat(runningPods).hasSize(2); - // we have two replicas, one is the HA leader - String exec = """ - kubectl get lease -n default spring-k8s-leader-election-lock \\ - -o "jsonpath={.spec.holderIdentity}" - """; - - String holderIdentity = container - .execInContainer("sh", "-c", exec) - .getStdout() - .trim(); - + String holderIdentity = currentLeaderAccordingToLeaderLease(container); assertThat(holderIdentity).isNotBlank(); assertThat(runningPods).contains(holderIdentity); } @@ -94,29 +90,94 @@ void persistsResourceVersionAndTriggersRefreshWithTwoReplicas(K3sContainer conta } }); - // 3. resource version of the configmap is present in out store - String initialResourceVersion = configMapResourceVersion(container); + // 3. resource version of the configmap is present in our store + String firstResourceVersion = configMapResourceVersion(container); Awaitilities.awaitUntilAsserted(120, 1000, - () -> assertThat(configMapResourceVersionInStateLease(container)) - .isEqualTo(initialResourceVersion)); + () -> { + Optional resourceVersionInStateLease = configMapResourceVersionInStateLease(container); + assertThat(resourceVersionInStateLease).isPresent(); + assertThat(resourceVersionInStateLease.get()).isEqualTo(firstResourceVersion); + }); - // 4. once we update the configmap, resourceVersion changes, and we have it in out store. + // 4. once we update the configmap, resourceVersion changes, and we have it in our store. WireMock.resetAllRequests(); - patchConfigMap(container); - String updatedResourceVersion = configMapResourceVersion(container); + patchConfigMap(container, "updated"); + String secondResourceVersion = configMapResourceVersion(container); // 5. because of the update in the configmap, watcher caught that and sent a // refresh call to the actuator ( wiremock in our test ) - TestUtil.verifyActuatorCalled(1); + Awaitilities.awaitUntilAsserted(120, 1000, () -> TestUtil.verifyActuatorCalled(1)); + WireMock.resetAllRequests(); // 6. the new resourceVersion is not equal to the previous one // 7. we have the latest resourceVersion in our store Awaitilities.awaitUntilAsserted(120, 1000, () -> { - String afterPatchResourceVersion = configMapResourceVersionInStateLease(container); - assertThat(afterPatchResourceVersion).isNotEqualTo(initialResourceVersion); - assertThat(afterPatchResourceVersion).isEqualTo(updatedResourceVersion); + Optional afterPatchResourceVersion = configMapResourceVersionInStateLease(container); + assertThat(afterPatchResourceVersion).isPresent(); + assertThat(afterPatchResourceVersion.get()).isNotEqualTo(firstResourceVersion); + assertThat(afterPatchResourceVersion.get()).isEqualTo(secondResourceVersion); }); + + // 8. delete the current leader and wait until it is gone. + String firstLeader = currentLeaderAccordingToLeaderLease(container); + deletePod(container, firstLeader); + Awaitilities.awaitUntilAsserted(120, 1000, () -> { + // pods do not contain the leader anymore ( we have removed it ) + assertThat(runningPods(container)).doesNotContain(firstLeader); + // but the lease still holds the pod that was removed ( since the lease has not expired yet ) + assertThat(currentLeaderAccordingToLeaderLease(container)).isEqualTo(firstLeader); + }); + // from the moment the above assertions pass, we have roughly 15 seconds + // before a new pod acquires the leadership ( this is lease-duration ) + // within this time we need to patch configmap and make a few assertions + // before a new leader is established + + + // 9. update configmap while there is no actual leader established + // resourceVersion is incremented in k8s, but we do not store it + patchConfigMap(container, "updated-after-leader-loss"); + String thirdResourceVersion = configMapResourceVersion(container); + + // resourceVersion has incremented in k8s + assertThat(thirdResourceVersion).isNotEqualTo(secondResourceVersion); + // but it stays the previous one in the state store + assertThat(configMapResourceVersionInStateLease(container)).contains(secondResourceVersion); + + // since there is no config watcher leader, refresh does not happen, since no one triggered it + WireMock.verify(WireMock.exactly(0), WireMock.postRequestedFor(WireMock.urlEqualTo("/actuator/refresh"))); + + // 10. leadership is again established, the resourceVersion that we missed is delivered to us + // and the refresh is triggered + Awaitilities.awaitUntilAsserted(120, 1000, () -> { + String secondLeader = currentLeaderAccordingToLeaderLease(container); + assertThat(secondLeader).isNotBlank().isNotEqualTo(firstLeader); + assertThat(runningPods(container)).contains(secondLeader); + }); + + Awaitilities.awaitUntilAsserted(120, 1000, () -> TestUtil.verifyActuatorCalled(1)); + + // 11. the resource version from the replayed event is stored in the HA Lease. + Awaitilities.awaitUntilAsserted(120, 1000, + () -> assertThat(configMapResourceVersionInStateLease(container)) + .contains(thirdResourceVersion)); + } + + private String currentLeaderAccordingToLeaderLease(K3sContainer container) { + String exec = """ + kubectl get lease -n default spring-k8s-leader-election-lock \\ + -o "jsonpath={.spec.holderIdentity}" + """; + + try { + return container + .execInContainer("sh", "-c", exec) + .getStdout() + .trim(); + } + catch (Exception e) { + throw new RuntimeException(e); + } } /** @@ -165,7 +226,7 @@ private String configMapResourceVersion(K3sContainer container) { } } - private String configMapResourceVersionInStateLease(K3sContainer container) { + private Optional configMapResourceVersionInStateLease(K3sContainer container) { String exec = """ kubectl get lease -n default configuration-watcher-ha \\ @@ -173,26 +234,49 @@ private String configMapResourceVersionInStateLease(K3sContainer container) { """; try { + // default=123 String storedResourceVersion = container .execInContainer("sh", "-c", exec) .getStdout() .trim(); // get only the 123 part - return storedResourceVersion.substring("default=".length()); + + if (!storedResourceVersion.trim().isEmpty()) { + return Optional.of(storedResourceVersion.substring("default=".length())); + } + + return Optional.empty(); + } catch (Exception e) { throw new RuntimeException(e); } } - private void patchConfigMap(K3sContainer container) { + private void patchConfigMap(K3sContainer container, String value) { String exec = """ kubectl patch configmap service-wiremock -n default --type merge \\ - -p '{"data":{"foo":"updated"}}' - """; + -p '{"data":{"foo":"%s"}}' + """.formatted(value); + + try { + container.execInContainer("sh", "-c", exec); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + // Kill the pod without graceful shutdown. + // This prevents the leader-election code from releasing the Lease. + // The next replica must wait for the Lease to expire before acquiring leadership. + // so we wait until it is removed, but do not --force it. + private void deletePod(K3sContainer container, String podName) { try { + String exec = """ + kubectl delete pod -n default ${podName} --grace-period=0 --wait=true + """.replace("${podName}", podName); container.execInContainer("sh", "-c", exec); } catch (Exception e) { From 4b9b5b1c9fadc9167dfeefa51b9a6d1e48f64e95 Mon Sep 17 00:00:00 2001 From: wind57 Date: Wed, 26 Aug 2026 13:11:22 +0300 Subject: [PATCH 17/24] tighther IT Signed-off-by: wind57 --- ...ientEventBasedConfigMapChangeDetector.java | 33 +++-- ...ClientEventBasedSecretsChangeDetector.java | 25 +++- ...ernetesClientConfigurationWatcherHaIT.java | 118 ++++++++---------- 3 files changed, 96 insertions(+), 80 deletions(-) diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java index 2d4f1dc965..45bd130566 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java @@ -143,22 +143,35 @@ public final void start(Map storedResourceVersions, labelSelector = configMapsLabels; } + ConfigMapResourceEventHandler handler = new ConfigMapResourceEventHandler(this::onEvent, resourceVersionWriter); + namespaces.forEach(namespace -> { SharedIndexInformer informer; - ConfigMapResourceEventHandler handler = new ConfigMapResourceEventHandler(this::onEvent, - resourceVersionWriter); SharedInformerFactory factory = new SharedInformerFactory(apiClient); factories.add(factory); - informer = factory - .sharedIndexInformerFor((CallGeneratorParams params) -> coreV1Api.listNamespacedConfigMap(namespace) + informer = factory.sharedIndexInformerFor((CallGeneratorParams params) -> { + + String resourceVersion = resourceVersionResolver.resolve(namespace, params.resourceVersion); + var request = coreV1Api.listNamespacedConfigMap(namespace) .timeoutSeconds(params.timeoutSeconds) - .resourceVersion(resourceVersionResolver.resolve(namespace, params.resourceVersion)) + .resourceVersion(resourceVersion) .watch(params.watch) - .labelSelector(labelSelector(labelSelector)) - .buildCall(null), V1ConfigMap.class, V1ConfigMapList.class); - - LOG.debug( - () -> "added configmap informer for namespace : " + namespace + " with labels : " + labelSelector); + .labelSelector(labelSelector(labelSelector)); + + // The stored resource version is the last checkpoint processed by the + // previous + // leader. Restore the informer from exactly that snapshot so its + // following WATCH requests + // start at the same version and can deliver every change after the + // checkpoint. + if (!params.watch && params.resourceVersion == null && resourceVersion != null) { + request.resourceVersionMatch("Exact"); + } + + return request.buildCall(null); + }, V1ConfigMap.class, V1ConfigMapList.class); + + LOG.debug(() -> "add configmap informer for namespace : " + namespace + " with labels : " + labelSelector); informer.addEventHandler(handler); informers.add(informer); diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java index 21b0da28e9..ed1d308360 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java @@ -146,18 +146,31 @@ public final void start(Map storedResourceVersions, labelSelector = secretsLabels; } + SecretResourceEventHandler handler = new SecretResourceEventHandler(this::onEvent, resourceVersionWriter); namespaces.forEach(namespace -> { SharedIndexInformer informer; - SecretResourceEventHandler handler = new SecretResourceEventHandler(this::onEvent, resourceVersionWriter); SharedInformerFactory factory = new SharedInformerFactory(apiClient); factories.add(factory); - informer = factory - .sharedIndexInformerFor((CallGeneratorParams params) -> coreV1Api.listNamespacedSecret(namespace) + informer = factory.sharedIndexInformerFor((CallGeneratorParams params) -> { + + String resourceVersion = resourceVersionResolver.resolve(namespace, params.resourceVersion); + var request = coreV1Api.listNamespacedSecret(namespace) .timeoutSeconds(params.timeoutSeconds) - .resourceVersion(resourceVersionResolver.resolve(namespace, params.resourceVersion)) + .resourceVersion(resourceVersion) .watch(params.watch) - .labelSelector(labelSelector(labelSelector)) - .buildCall(null), V1Secret.class, V1SecretList.class); + .labelSelector(labelSelector(labelSelector)); + + // The stored resource version is the last checkpoint processed by the + // previous + // leader. Restore the informer from exactly that snapshot so its + // following WATCH requests + // start at the same version and can deliver every change after the + // checkpoint. + if (!params.watch && params.resourceVersion == null && resourceVersion != null) { + request.resourceVersionMatch("Exact"); + } + return request.buildCall(null); + }, V1Secret.class, V1SecretList.class); LOG.debug(() -> "secret informer for namespace : " + namespace + " with filter : " + secretsLabels); diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java index f3a4fe7c6f..4de4bafe9a 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java @@ -37,10 +37,10 @@ * * @author wind57 */ - @NativeClientIntegrationTest(withImages = { "spring-cloud-kubernetes-configuration-watcher" }, - wiremock = @NativeClientIntegrationTest.Wiremock(enabled = true, namespaces = "default", withNodePort = true), - rbacNamespaces = "default", - configurationWatcher = @NativeClientIntegrationTest.ConfigurationWatcher(enabled = true, enableHa = true, +@NativeClientIntegrationTest(withImages = { "spring-cloud-kubernetes-configuration-watcher" }, + wiremock = @NativeClientIntegrationTest.Wiremock(enabled = true, namespaces = "default", withNodePort = true), + rbacNamespaces = "default", + configurationWatcher = @NativeClientIntegrationTest.ConfigurationWatcher(enabled = true, enableHa = true, replicas = 2, refreshDelay = "0", reloadEnabled = false)) class KubernetesClientConfigurationWatcherHaIT { @@ -92,32 +92,34 @@ void persistsResourceVersionAndReplaysChangeAfterLeaderLoss(K3sContainer contain // 3. resource version of the configmap is present in our store String firstResourceVersion = configMapResourceVersion(container); - Awaitilities.awaitUntilAsserted(120, 1000, - () -> { - Optional resourceVersionInStateLease = configMapResourceVersionInStateLease(container); - assertThat(resourceVersionInStateLease).isPresent(); - assertThat(resourceVersionInStateLease.get()).isEqualTo(firstResourceVersion); - }); + Awaitilities.awaitUntilAsserted(120, 1000, () -> { + Optional resourceVersionInStateLease = configMapResourceVersionInStateLease(container); + assertThat(resourceVersionInStateLease).isPresent(); + assertThat(resourceVersionInStateLease.get()).isEqualTo(firstResourceVersion); + }); - // 4. once we update the configmap, resourceVersion changes, and we have it in our store. + // 4. once we update the configmap, resourceVersion changes, and we have it in our + // store. + // Wait for the initial onAdd refresh to complete. + Awaitilities.awaitUntilAsserted(120, 1000, () -> TestUtil.verifyActuatorCalled(1)); + // ignore initial onAdd refresh WireMock.resetAllRequests(); + patchConfigMap(container, "updated"); String secondResourceVersion = configMapResourceVersion(container); // 5. because of the update in the configmap, watcher caught that and sent a // refresh call to the actuator ( wiremock in our test ) Awaitilities.awaitUntilAsserted(120, 1000, () -> TestUtil.verifyActuatorCalled(1)); - WireMock.resetAllRequests(); // 6. the new resourceVersion is not equal to the previous one // 7. we have the latest resourceVersion in our store - Awaitilities.awaitUntilAsserted(120, 1000, - () -> { - Optional afterPatchResourceVersion = configMapResourceVersionInStateLease(container); - assertThat(afterPatchResourceVersion).isPresent(); - assertThat(afterPatchResourceVersion.get()).isNotEqualTo(firstResourceVersion); - assertThat(afterPatchResourceVersion.get()).isEqualTo(secondResourceVersion); - }); + Awaitilities.awaitUntilAsserted(120, 1000, () -> { + Optional afterPatchResourceVersion = configMapResourceVersionInStateLease(container); + assertThat(afterPatchResourceVersion).isPresent(); + assertThat(afterPatchResourceVersion.get()).isNotEqualTo(firstResourceVersion); + assertThat(afterPatchResourceVersion.get()).isEqualTo(secondResourceVersion); + }); // 8. delete the current leader and wait until it is gone. String firstLeader = currentLeaderAccordingToLeaderLease(container); @@ -125,7 +127,8 @@ void persistsResourceVersionAndReplaysChangeAfterLeaderLoss(K3sContainer contain Awaitilities.awaitUntilAsserted(120, 1000, () -> { // pods do not contain the leader anymore ( we have removed it ) assertThat(runningPods(container)).doesNotContain(firstLeader); - // but the lease still holds the pod that was removed ( since the lease has not expired yet ) + // but the lease still holds the pod that was removed ( since the lease has + // not expired yet ) assertThat(currentLeaderAccordingToLeaderLease(container)).isEqualTo(firstLeader); }); // from the moment the above assertions pass, we have roughly 15 seconds @@ -133,9 +136,11 @@ void persistsResourceVersionAndReplaysChangeAfterLeaderLoss(K3sContainer contain // within this time we need to patch configmap and make a few assertions // before a new leader is established - // 9. update configmap while there is no actual leader established // resourceVersion is incremented in k8s, but we do not store it + // also there is no leader to react to the patch in the configmap, so no actuator + // call + WireMock.resetAllRequests(); patchConfigMap(container, "updated-after-leader-loss"); String thirdResourceVersion = configMapResourceVersion(container); @@ -144,10 +149,8 @@ void persistsResourceVersionAndReplaysChangeAfterLeaderLoss(K3sContainer contain // but it stays the previous one in the state store assertThat(configMapResourceVersionInStateLease(container)).contains(secondResourceVersion); - // since there is no config watcher leader, refresh does not happen, since no one triggered it - WireMock.verify(WireMock.exactly(0), WireMock.postRequestedFor(WireMock.urlEqualTo("/actuator/refresh"))); - - // 10. leadership is again established, the resourceVersion that we missed is delivered to us + // 10. leadership is again established, the resourceVersion that we missed is + // delivered to us // and the refresh is triggered Awaitilities.awaitUntilAsserted(120, 1000, () -> { String secondLeader = currentLeaderAccordingToLeaderLease(container); @@ -159,21 +162,17 @@ void persistsResourceVersionAndReplaysChangeAfterLeaderLoss(K3sContainer contain // 11. the resource version from the replayed event is stored in the HA Lease. Awaitilities.awaitUntilAsserted(120, 1000, - () -> assertThat(configMapResourceVersionInStateLease(container)) - .contains(thirdResourceVersion)); + () -> assertThat(configMapResourceVersionInStateLease(container)).contains(thirdResourceVersion)); } private String currentLeaderAccordingToLeaderLease(K3sContainer container) { String exec = """ - kubectl get lease -n default spring-k8s-leader-election-lock \\ - -o "jsonpath={.spec.holderIdentity}" - """; + kubectl get lease -n default spring-k8s-leader-election-lock \\ + -o "jsonpath={.spec.holderIdentity}" + """; try { - return container - .execInContainer("sh", "-c", exec) - .getStdout() - .trim(); + return container.execInContainer("sh", "-c", exec).getStdout().trim(); } catch (Exception e) { throw new RuntimeException(e); @@ -186,21 +185,18 @@ private String currentLeaderAccordingToLeaderLease(K3sContainer container) { private List runningPods(K3sContainer container) { String exec = """ - kubectl get pods -n default -l app=spring-cloud-kubernetes-configuration-watcher \\ - --field-selector=status.phase=Running \\ - -o "jsonpath={.items[*].metadata.name}" - """; + kubectl get pods -n default -l app=spring-cloud-kubernetes-configuration-watcher \\ + --field-selector=status.phase=Running \\ + -o "jsonpath={.items[*].metadata.name}" + """; try { - String runningPods = container - .execInContainer("sh", "-c", exec) - .getStdout() - .trim(); + String runningPods = container.execInContainer("sh", "-c", exec).getStdout().trim(); - return runningPods.isEmpty() ? List.of() - : Arrays.stream(runningPods.split("\\s+")).toList(); + return runningPods.isEmpty() ? List.of() : Arrays.stream(runningPods.split("\\s+")).toList(); - } catch (Exception e) { + } + catch (Exception e) { throw new RuntimeException(e); } } @@ -211,15 +207,12 @@ private List runningPods(K3sContainer container) { private String configMapResourceVersion(K3sContainer container) { String exec = """ - kubectl get configmap service-wiremock -n default \\ - -o "jsonpath={.metadata.resourceVersion}" - """; + kubectl get configmap service-wiremock -n default \\ + -o "jsonpath={.metadata.resourceVersion}" + """; try { - return container - .execInContainer("sh", "-c", exec) - .getStdout() - .trim(); + return container.execInContainer("sh", "-c", exec).getStdout().trim(); } catch (Exception e) { throw new RuntimeException(e); @@ -229,17 +222,14 @@ private String configMapResourceVersion(K3sContainer container) { private Optional configMapResourceVersionInStateLease(K3sContainer container) { String exec = """ - kubectl get lease -n default configuration-watcher-ha \\ - -o "jsonpath={.metadata.annotations['spring\\.cloud\\.kubernetes\\.configuration\\.watcher/configmap-resource-version']}" - """; + kubectl get lease -n default configuration-watcher-ha \\ + -o "jsonpath={.metadata.annotations['spring\\.cloud\\.kubernetes\\.configuration\\.watcher/configmap-resource-version']}" + """; try { // default=123 - String storedResourceVersion = container - .execInContainer("sh", "-c", exec) - .getStdout() - .trim(); + String storedResourceVersion = container.execInContainer("sh", "-c", exec).getStdout().trim(); // get only the 123 part if (!storedResourceVersion.trim().isEmpty()) { @@ -256,9 +246,9 @@ private Optional configMapResourceVersionInStateLease(K3sContainer conta private void patchConfigMap(K3sContainer container, String value) { String exec = """ - kubectl patch configmap service-wiremock -n default --type merge \\ - -p '{"data":{"foo":"%s"}}' - """.formatted(value); + kubectl patch configmap service-wiremock -n default --type merge \\ + -p '{"data":{"foo":"%s"}}' + """.formatted(value); try { container.execInContainer("sh", "-c", exec); @@ -275,8 +265,8 @@ private void patchConfigMap(K3sContainer container, String value) { private void deletePod(K3sContainer container, String podName) { try { String exec = """ - kubectl delete pod -n default ${podName} --grace-period=0 --wait=true - """.replace("${podName}", podName); + kubectl delete pod -n default ${podName} --grace-period=0 --wait=true + """.replace("${podName}", podName); container.execInContainer("sh", "-c", exec); } catch (Exception e) { From fa23cb00981827a1f6bdc44cfed28a2671d39f10 Mon Sep 17 00:00:00 2001 From: wind57 Date: Wed, 26 Aug 2026 13:17:21 +0300 Subject: [PATCH 18/24] tighther IT Signed-off-by: wind57 --- .../watcher/KubernetesClientConfigurationWatcherHaIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java index 4de4bafe9a..b9712ac88c 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/KubernetesClientConfigurationWatcherHaIT.java @@ -149,6 +149,10 @@ void persistsResourceVersionAndReplaysChangeAfterLeaderLoss(K3sContainer contain // but it stays the previous one in the state store assertThat(configMapResourceVersionInStateLease(container)).contains(secondResourceVersion); + // no leader was active, so this update must not have triggered an actuator call + WireMock.verify(WireMock.exactly(0), + WireMock.postRequestedFor(WireMock.urlEqualTo("/actuator/refresh"))); + // 10. leadership is again established, the resourceVersion that we missed is // delivered to us // and the refresh is triggered From f48e141c986a12f3057249ba875ebc10028ac48a Mon Sep 17 00:00:00 2001 From: wind57 Date: Wed, 26 Aug 2026 13:48:01 +0300 Subject: [PATCH 19/24] add documentation Signed-off-by: wind57 --- ...loud-kubernetes-configuration-watcher.adoc | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docs/modules/ROOT/pages/spring-cloud-kubernetes-configuration-watcher.adoc b/docs/modules/ROOT/pages/spring-cloud-kubernetes-configuration-watcher.adoc index 3b6378560d..df2f16d353 100644 --- a/docs/modules/ROOT/pages/spring-cloud-kubernetes-configuration-watcher.adoc +++ b/docs/modules/ROOT/pages/spring-cloud-kubernetes-configuration-watcher.adoc @@ -98,6 +98,74 @@ This tells how many milliseconds should we wait before firing the event from con You need to "match" this _eventually_ part to that value in milliseconds on your cluster. +## High Availability + +The Configuration Watcher can run with multiple replicas, with only one replica actively +watching ConfigMaps and Secrets at a time. High availability is disabled by default. Enable +both the native Kubernetes leader election and Configuration Watcher HA: + +[source,yaml] +---- +spring: + cloud: + kubernetes: + leader: + election: + enabled: true + configuration: + watcher: + ha: + enabled: true +---- + +The two settings have different responsibilities: + +* `spring.cloud.kubernetes.leader.election.enabled` enables the native Kubernetes leader + election used to choose the active watcher replica. +* `spring.cloud.kubernetes.configuration.watcher.ha.enabled` makes the Configuration Watcher + start its ConfigMap and Secret informers only after it becomes the leader, and stop them when + it loses leadership. + +The leader-election lock and the Configuration Watcher HA state are stored separately. The +Configuration Watcher state is persisted in a Kubernetes `Lease`. Its defaults are: + +* `spring.cloud.kubernetes.configuration.watcher.ha.lease-name`: `configuration-watcher-ha` +* `spring.cloud.kubernetes.configuration.watcher.ha.lease-namespace`: `default` + +The state lease stores the last processed informer resource version separately for each watched +namespace and resource type. When a new replica becomes the leader, it reads those checkpoints +and starts each informer from its stored resource version. This allows changes observed while no +watcher was leader to be delivered to the new leader. The watcher does not refresh every resource +after a leadership change; it only replays events for the configured ConfigMap and Secret +informers. + +Processing is at least once. If a watcher loses leadership after an event was delivered but before +its checkpoint was persisted, the replacement leader may process that event again. Refresh targets +must therefore tolerate repeated refresh notifications. + +The service account used by the Configuration Watcher needs permission to read and update the HA +state lease in the configured lease namespace. Add the following rule to the Role in that +namespace: + +[source,yaml] +---- +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "create", "update"] +---- + +The HA state lease defaults to the `default` namespace even when the Configuration Watcher runs +in another namespace. If you configure a different `lease-namespace`, create the corresponding +Role and RoleBinding there, or grant equivalent cluster-scoped permissions. + +Configure at least two replicas in the Configuration Watcher Deployment to enable failover: + +[source,yaml] +---- +spec: + replicas: 2 +---- + Spring Cloud Kubernetes Configuration Watcher can send refresh notifications to applications in two ways. 1. Over HTTP, in which case the application being notified, must have the `/refresh` actuator endpoint exposed and accessible from within the cluster @@ -155,6 +223,9 @@ items: - apiGroups: ["", "extensions", "apps"] resources: ["configmaps", "pods", "services", "endpoints", "secrets"] verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "create", "update"] - apiVersion: apps/v1 kind: Deployment metadata: From b1e283150bb749881da064970b3883d293342a1a Mon Sep 17 00:00:00 2001 From: wind57 Date: Wed, 26 Aug 2026 14:12:33 +0300 Subject: [PATCH 20/24] drop useless method Signed-off-by: wind57 --- .../KubernetesClientEventBasedSecretsChangeDetectorTests.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java index 3d6c5ea2e8..2e50264480 100644 --- a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java +++ b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java @@ -398,10 +398,6 @@ void equalsEight() { Assertions.assertThat(result).isFalse(); } - private void changeDetectorAssert() { - changeDetectorAssert(true); - } - private void changeDetectorAssert(boolean haEnabled) { // coreV1Api From 2f99763b0b8e3aacb04f9f04ac91ffb3fdf83796 Mon Sep 17 00:00:00 2001 From: wind57 Date: Wed, 26 Aug 2026 14:14:21 +0300 Subject: [PATCH 21/24] comment formatting Signed-off-by: wind57 --- ...KubernetesClientEventBasedConfigMapChangeDetector.java | 8 +++----- .../KubernetesClientEventBasedSecretsChangeDetector.java | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java index 45bd130566..b815d01420 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java @@ -159,11 +159,9 @@ public final void start(Map storedResourceVersions, .labelSelector(labelSelector(labelSelector)); // The stored resource version is the last checkpoint processed by the - // previous - // leader. Restore the informer from exactly that snapshot so its - // following WATCH requests - // start at the same version and can deliver every change after the - // checkpoint. + // previous leader. Restore the informer from exactly that snapshot so its + // following WATCH requests start at the same version and can deliver + // every change after the checkpoint. if (!params.watch && params.resourceVersion == null && resourceVersion != null) { request.resourceVersionMatch("Exact"); } diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java index ed1d308360..e9168c2518 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java @@ -161,11 +161,9 @@ public final void start(Map storedResourceVersions, .labelSelector(labelSelector(labelSelector)); // The stored resource version is the last checkpoint processed by the - // previous - // leader. Restore the informer from exactly that snapshot so its - // following WATCH requests - // start at the same version and can deliver every change after the - // checkpoint. + // previous leader. Restore the informer from exactly that snapshot so its + // following WATCH requests start at the same version and can deliver + // every change after the checkpoint. if (!params.watch && params.resourceVersion == null && resourceVersion != null) { request.resourceVersionMatch("Exact"); } From b9794b3c6952aa792c6590f9228220afda54089a Mon Sep 17 00:00:00 2001 From: wind57 Date: Wed, 26 Aug 2026 14:45:18 +0300 Subject: [PATCH 22/24] haEnabled short circuit Signed-off-by: wind57 --- .../KubernetesClientEventBasedConfigMapChangeDetector.java | 3 ++- .../KubernetesClientEventBasedSecretsChangeDetector.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java index b815d01420..01675f124d 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java @@ -162,7 +162,8 @@ public final void start(Map storedResourceVersions, // previous leader. Restore the informer from exactly that snapshot so its // following WATCH requests start at the same version and can deliver // every change after the checkpoint. - if (!params.watch && params.resourceVersion == null && resourceVersion != null) { + // we do not need haEnabled check here, but it short-circuits fast + if (haEnabled && !params.watch && params.resourceVersion == null && resourceVersion != null) { request.resourceVersionMatch("Exact"); } diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java index e9168c2518..5c8940da1f 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java @@ -164,7 +164,8 @@ public final void start(Map storedResourceVersions, // previous leader. Restore the informer from exactly that snapshot so its // following WATCH requests start at the same version and can deliver // every change after the checkpoint. - if (!params.watch && params.resourceVersion == null && resourceVersion != null) { + // we do not need haEnabled check here, but it short-circuits fast + if (haEnabled && !params.watch && params.resourceVersion == null && resourceVersion != null) { request.resourceVersionMatch("Exact"); } return request.buildCall(null); From 9548c44b7e00986d80fdbbc210b08e202cf83dc9 Mon Sep 17 00:00:00 2001 From: wind57 Date: Wed, 26 Aug 2026 17:53:34 +0300 Subject: [PATCH 23/24] fix ITs Signed-off-by: wind57 --- .../client/reload/it/K8sClientConfigMapEventTriggeredIT.java | 2 +- .../reload/it/K8sClientConfigMapLabelEventTriggeredIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientConfigMapEventTriggeredIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientConfigMapEventTriggeredIT.java index 5b69b6dc91..4e307d9085 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientConfigMapEventTriggeredIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientConfigMapEventTriggeredIT.java @@ -99,7 +99,7 @@ static void afterAllLocal(NativeClientKubernetesFixture fixture) { @Test void test(CapturedOutput output) { - assertReloadLogStatements("added configmap informer for namespace : right with labels : {}", + assertReloadLogStatements("add configmap informer for namespace : right with labels : {}", "added secret informer for namespace", output); Assertions.assertThat(rightProperties.getValue()).isEqualTo("right-initial"); diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientConfigMapLabelEventTriggeredIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientConfigMapLabelEventTriggeredIT.java index ac747d35a3..f04540587c 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientConfigMapLabelEventTriggeredIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-reload/src/test/java/org/springframework/cloud/kubernetes/k8s/client/reload/it/K8sClientConfigMapLabelEventTriggeredIT.java @@ -104,7 +104,7 @@ static void afterAllLocal(NativeClientKubernetesFixture fixture) { void test(CapturedOutput output) { assertReloadLogStatements( - "added configmap informer for namespace : " + "add configmap informer for namespace : " + "right with labels : {spring.cloud.kubernetes.config.informer.enabled=true}", "added secret informer for namespace", output); From 8a89ea9da16b032a7b62ff9ce37ac6c2a6d18ce4 Mon Sep 17 00:00:00 2001 From: wind57 Date: Fri, 28 Aug 2026 09:18:22 +0300 Subject: [PATCH 24/24] fix comments Signed-off-by: wind57 --- .../KubernetesClientEventBasedSecretsChangeDetector.java | 4 ++-- .../watcher/ha/LeaseConfigurationWatcherStateStore.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java index 5c8940da1f..3155e39b47 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java @@ -113,11 +113,11 @@ void inform() { // In HA mode, defer informer startup until this instance acquires leadership. // The leader callback restores the persisted state and then starts the informers. if (!haEnabled) { - LOG.info(() -> "config watcher HA is disabled : starting configmap informers immediately"); + LOG.info(() -> "config watcher HA is disabled : starting secret informers immediately"); start(Map.of(), null); } else { - LOG.info(() -> "config watcher HA is enabled : deferring configmap informer startup " + LOG.info(() -> "config watcher HA is enabled : deferring secret informer startup " + "until leadership is acquired"); } } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java index aa540b0b77..50db2e51b3 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ha/LeaseConfigurationWatcherStateStore.java @@ -113,7 +113,7 @@ private void createLease(String leaseName, String leaseNamespace) { api.createNamespacedLease(leaseNamespace, newLease(leaseName, leaseNamespace)).execute(); } catch (ApiException e) { - LOG.error(e, () -> "Failed to create watcher HA lease '" + e.getResponseBody()); + LOG.error(e, () -> "Failed to create watcher HA lease. " + e.getResponseBody()); throw new IllegalStateException( "Failed to create watcher HA lease '" + leaseName + "' in namespace '" + leaseNamespace + "'", e); }