diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java index 48a3fd54..f5f36591 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java @@ -18,6 +18,7 @@ package org.springframework.cloud.kubernetes.client.config.reload; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Set; import io.kubernetes.client.common.KubernetesObject; @@ -81,7 +82,12 @@ public class KubernetesClientEventBasedConfigMapChangeDetector extends Configura public void onUpdate(V1ConfigMap oldConfigMap, V1ConfigMap newConfigMap) { LOG.debug(() -> "ConfigMap " + newConfigMap.getMetadata().getName() + " was updated in namespace " + newConfigMap.getMetadata().getNamespace()); - onEvent(newConfigMap); + if (Objects.equals(oldConfigMap.getData(), newConfigMap.getData())) { + LOG.debug(() -> "data in configmap has not changed, will not reload"); + } + else { + onEvent(newConfigMap); + } } @Override diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java index bba31d69..2beb30db 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java @@ -17,7 +17,10 @@ package org.springframework.cloud.kubernetes.client.config.reload; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.Set; import io.kubernetes.client.common.KubernetesObject; @@ -81,7 +84,13 @@ public class KubernetesClientEventBasedSecretsChangeDetector extends Configurati public void onUpdate(V1Secret oldSecret, V1Secret newSecret) { LOG.debug(() -> "Secret " + newSecret.getMetadata().getName() + " was updated in namespace " + newSecret.getMetadata().getNamespace()); - onEvent(newSecret); + + if (KubernetesClientEventBasedSecretsChangeDetector.equals(oldSecret.getData(), newSecret.getData())) { + LOG.debug(() -> "data in secret has not changed, will not reload"); + } + else { + onEvent(newSecret); + } } @Override @@ -146,4 +155,22 @@ public class KubernetesClientEventBasedSecretsChangeDetector extends Configurati } } + static boolean equals(Map left, Map right) { + Map innerLeft = Optional.ofNullable(left).orElse(Map.of()); + Map innerRight = Optional.ofNullable(right).orElse(Map.of()); + + if (innerLeft.size() != innerRight.size()) { + return false; + } + + for (Map.Entry entry : innerLeft.entrySet()) { + String key = entry.getKey(); + byte[] value = entry.getValue(); + if (!Arrays.equals(value, innerRight.get(key))) { + return false; + } + } + return true; + } + } diff --git a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java index a71c5689..c4fa1d32 100644 --- a/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java +++ b/spring-cloud-kubernetes-client-config/src/test/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetectorTests.java @@ -16,22 +16,16 @@ package org.springframework.cloud.kubernetes.client.config.reload; -import java.io.IOException; -import java.lang.reflect.Modifier; import java.time.Duration; -import java.time.OffsetDateTime; import java.util.Base64; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.client.WireMock; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.TypeAdapter; -import com.google.gson.stream.JsonReader; -import com.google.gson.stream.JsonWriter; +import com.github.tomakehurst.wiremock.matching.StringValuePattern; import io.kubernetes.client.informer.EventType; import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.JSON; @@ -45,6 +39,7 @@ import io.kubernetes.client.util.Watch; import okhttp3.OkHttpClient; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -71,6 +66,12 @@ import static org.mockito.Mockito.when; */ class KubernetesClientEventBasedSecretsChangeDetectorTests { + private static final Map WATCH_FALSE = Map.of("watch", equalTo("false")); + + private static final Map WATCH_TRUE = Map.of("watch", equalTo("true")); + + private static final String SCENARIO = "watch"; + private static WireMockServer wireMockServer; @BeforeAll @@ -93,43 +94,45 @@ class KubernetesClientEventBasedSecretsChangeDetectorTests { @Test void watch() { - GsonBuilder builder = new GsonBuilder(); - builder.excludeFieldsWithModifiers(Modifier.STATIC, Modifier.TRANSIENT, Modifier.VOLATILE) - .registerTypeAdapter(OffsetDateTime.class, new GsonOffsetDateTimeAdapter()); - Gson gson = builder.create(); - V1Secret dbPassword = new V1Secret().kind("Secret").metadata(new V1ObjectMeta().name("db-password")) + V1Secret dbPassword = new V1Secret().metadata(new V1ObjectMeta().name("db-password")) .putStringDataItem("password", Base64.getEncoder().encodeToString("p455w0rd".getBytes())) - .putStringDataItem("username", Base64.getEncoder().encodeToString("user".getBytes())); - V1Secret dbPasswordUpdated = new V1Secret().kind("Secret").metadata(new V1ObjectMeta().name("db-password")) - .putStringDataItem("password", Base64.getEncoder().encodeToString("p455w0rd2".getBytes())) - .putStringDataItem("username", Base64.getEncoder().encodeToString("user".getBytes())); - V1SecretList secretList = new V1SecretList().kind("SecretList").metadata(new V1ListMeta().resourceVersion("0")) + .putDataItem("password", Base64.getEncoder().encode("p455w0rd".getBytes())) + .putStringDataItem("username", Base64.getEncoder().encodeToString("user".getBytes())) + .putDataItem("username", Base64.getEncoder().encode("user".getBytes())); + V1SecretList secretList = new V1SecretList().metadata(new V1ListMeta().resourceVersion("0")) .items(List.of(dbPassword)); - stubFor(get(urlMatching("^/api/v1/namespaces/default/secrets.*")).inScenario("watch") - .whenScenarioStateIs(STARTED).withQueryParam("watch", equalTo("false")) - .willReturn(aResponse().withStatus(200).withBody(gson.toJson(secretList))).willSetStateTo("update")); - + V1Secret dbPasswordUpdated = new V1Secret().metadata(new V1ObjectMeta().name("db-password")) + .putStringDataItem("password", Base64.getEncoder().encodeToString("p455w0rd2".getBytes())) + .putDataItem("password", Base64.getEncoder().encode("p455w0rd2".getBytes())) + .putStringDataItem("username", Base64.getEncoder().encodeToString("user".getBytes())) + .putDataItem("username", Base64.getEncoder().encode("user".getBytes())); Watch.Response watchResponse = new Watch.Response<>(EventType.MODIFIED.name(), dbPasswordUpdated); - stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).inScenario("watch") - .whenScenarioStateIs("update").withQueryParam("watch", equalTo("true")) + + stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).inScenario(SCENARIO) + .whenScenarioStateIs(STARTED).withQueryParams(WATCH_FALSE) + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(secretList))) + .willSetStateTo("update")); + + stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).inScenario(SCENARIO) + .whenScenarioStateIs("update").withQueryParams(WATCH_TRUE) .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(watchResponse))) .willSetStateTo("add")); - stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).inScenario("watch").whenScenarioStateIs("add") - .withQueryParam("watch", equalTo("true")) + stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).inScenario(SCENARIO).whenScenarioStateIs("add") + .withQueryParams(WATCH_TRUE) .willReturn(aResponse().withStatus(200) .withBody(new JSON().serialize(new Watch.Response<>(EventType.ADDED.name(), - new V1Secret().kind("Secret").metadata(new V1ObjectMeta().name("rabbit-password")) + new V1Secret().metadata(new V1ObjectMeta().name("rabbit-password")) .putDataItem("rabbit-pw", Base64.getEncoder().encode("password".getBytes())))))) .willSetStateTo("delete")); - stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).inScenario("watch") - .whenScenarioStateIs("delete").withQueryParam("watch", equalTo("true")) + stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).inScenario(SCENARIO) + .whenScenarioStateIs("delete").withQueryParams(WATCH_TRUE) .willReturn(aResponse().withStatus(200) .withBody(new JSON().serialize(new Watch.Response<>(EventType.DELETED.name(), - new V1Secret().kind("Secret").metadata(new V1ObjectMeta().name("rabbit-password")) + new V1Secret().metadata(new V1ObjectMeta().name("rabbit-password")) .putDataItem("rabbit-pw", Base64.getEncoder().encode("password".getBytes())))))) .willSetStateTo("done")); @@ -167,21 +170,121 @@ class KubernetesClientEventBasedSecretsChangeDetectorTests { await().timeout(Duration.ofSeconds(10)).pollInterval(Duration.ofSeconds(2)).until(() -> howMany[0] >= 4); } - // This is needed when using JDK17 because GSON uses reflection to construct an - // OffsetDateTime but that constructor - // is protected. - public final static class GsonOffsetDateTimeAdapter extends TypeAdapter { + /** + * both are null, treat that as no change. + */ + @Test + void equalsOne() { + Map left = null; + Map right = null; - @Override - public void write(JsonWriter jsonWriter, OffsetDateTime localDateTime) throws IOException { - jsonWriter.value(OffsetDateTime.now().toString()); - } + boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right); + Assertions.assertTrue(result); + } - @Override - public OffsetDateTime read(JsonReader jsonReader) { - return OffsetDateTime.now(); - } + /** + * - left is empty map + * - right is null + * + * treat as equal, that is: no change + */ + @Test + void equalsTwo() { + Map left = Map.of(); + Map right = null; + boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right); + Assertions.assertTrue(result); + } + + /** + * - left is empty map + * - right is null + * + * treat as equal, that is: no change + */ + @Test + void equalsThree() { + Map left = Map.of(); + Map right = null; + + boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right); + Assertions.assertTrue(result); + } + + /** + * - left is null + * - right is empty map + * + * treat as equal, that is: no change + */ + @Test + void equalsFour() { + Map left = null; + Map right = Map.of(); + + boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right); + Assertions.assertTrue(result); + } + + /** + * - left is empty map + * - right is empty map + * + * treat as equal, that is: no change + */ + @Test + void equalsFive() { + Map left = Map.of(); + Map right = Map.of(); + + boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right); + Assertions.assertTrue(result); + } + + /** + * - left is empty map + * - right is [1, b] + * + * treat as non-equal, that is change + */ + @Test + void equalsSix() { + Map left = Map.of(); + Map right = Map.of("1", "b".getBytes()); + + boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right); + Assertions.assertFalse(result); + } + + /** + * - left is [1, a] + * - right is [1, b] + * + * treat as non-equal, that is change + */ + @Test + void equalsSeven() { + Map left = Map.of("1", "a".getBytes()); + Map right = Map.of("1", "b".getBytes()); + + boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right); + Assertions.assertFalse(result); + } + + /** + * - left is [1, a, 2 aa] + * - right is [1, b, 2, aa] + * + * treat as non-equal, that is change + */ + @Test + void equalsEight() { + Map left = Map.of("1", "a".getBytes(), "2", "aa".getBytes()); + Map right = Map.of("1", "b".getBytes(), "2", "aa".getBytes()); + + boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right); + Assertions.assertFalse(result); } } diff --git a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/config/reload/ConfigReloadUtil.java b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/config/reload/ConfigReloadUtil.java index fe77e472..96590176 100644 --- a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/config/reload/ConfigReloadUtil.java +++ b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/config/reload/ConfigReloadUtil.java @@ -52,11 +52,11 @@ public final class ConfigReloadUtil { boolean changed = changed(sourceFromK8s, existingSources); if (changed) { - LOG.info("Detected change in config maps"); + LOG.info("Detected change in config maps/secrets"); return true; } else { - LOG.debug("No change detected in config maps, reload will not happen"); + LOG.debug("No change detected in config maps/secrets, reload will not happen"); } return false; diff --git a/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedConfigMapChangeDetector.java b/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedConfigMapChangeDetector.java index fdb91e98..fef7de6f 100644 --- a/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedConfigMapChangeDetector.java +++ b/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedConfigMapChangeDetector.java @@ -19,6 +19,7 @@ package org.springframework.cloud.kubernetes.fabric8.config.reload; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import io.fabric8.kubernetes.api.model.ConfigMap; @@ -131,7 +132,12 @@ public class Fabric8EventBasedConfigMapChangeDetector extends ConfigurationChang public void onUpdate(ConfigMap oldConfigMap, ConfigMap newConfigMap) { LOG.debug("ConfigMap " + newConfigMap.getMetadata().getName() + " was updated in namespace " + newConfigMap.getMetadata().getNamespace()); - onEvent(newConfigMap); + if (Objects.equals(oldConfigMap.getData(), newConfigMap.getData())) { + LOG.debug(() -> "data in configmap has not changed, will not reload"); + } + else { + onEvent(newConfigMap); + } } @Override diff --git a/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedSecretsChangeDetector.java b/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedSecretsChangeDetector.java index 74615e57..325b0ba0 100644 --- a/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedSecretsChangeDetector.java +++ b/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedSecretsChangeDetector.java @@ -19,6 +19,7 @@ package org.springframework.cloud.kubernetes.fabric8.config.reload; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import io.fabric8.kubernetes.api.model.Secret; @@ -133,7 +134,12 @@ public class Fabric8EventBasedSecretsChangeDetector extends ConfigurationChangeD public void onUpdate(Secret oldSecret, Secret newSecret) { LOG.debug("Secret " + newSecret.getMetadata().getName() + " was updated in namespace " + newSecret.getMetadata().getNamespace()); - onEvent(newSecret); + if (Objects.equals(oldSecret.getData(), newSecret.getData())) { + LOG.debug(() -> "data in secret has not changed, will not reload"); + } + else { + onEvent(newSecret); + } } @Override diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-configmap-event-reload/src/test/java/org/springframework/cloud/kubernetes/client/configmap/event/reload/DataChangesInConfigMapReloadIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-configmap-event-reload/src/test/java/org/springframework/cloud/kubernetes/client/configmap/event/reload/DataChangesInConfigMapReloadIT.java new file mode 100644 index 00000000..b3f112de --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-configmap-event-reload/src/test/java/org/springframework/cloud/kubernetes/client/configmap/event/reload/DataChangesInConfigMapReloadIT.java @@ -0,0 +1,216 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.configmap.event.reload; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1ConfigMap; +import io.kubernetes.client.openapi.models.V1ConfigMapBuilder; +import io.kubernetes.client.openapi.models.V1Deployment; +import io.kubernetes.client.openapi.models.V1EnvVar; +import io.kubernetes.client.openapi.models.V1Ingress; +import io.kubernetes.client.openapi.models.V1ObjectMetaBuilder; +import io.kubernetes.client.openapi.models.V1Service; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.Container; +import org.testcontainers.k3s.K3sContainer; +import reactor.netty.http.client.HttpClient; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; +import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; +import org.springframework.cloud.kubernetes.integration.tests.commons.native_client.Util; +import org.springframework.http.HttpMethod; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; +import org.springframework.web.reactive.function.client.WebClient; + +import static org.awaitility.Awaitility.await; + +class DataChangesInConfigMapReloadIT { + + private static final String IMAGE_NAME = "spring-cloud-kubernetes-client-configmap-event-reload"; + + private static final String NAMESPACE = "default"; + + private static final String LEFT_NAMESPACE = "left"; + + private static final K3sContainer K3S = Commons.container(); + + private static Util util; + + private static CoreV1Api api; + + @BeforeAll + static void beforeAll() throws Exception { + K3S.start(); + Commons.validateImage(IMAGE_NAME, K3S); + Commons.loadSpringCloudKubernetesImage(IMAGE_NAME, K3S); + + util = new Util(K3S); + api = new CoreV1Api(); + + util.createNamespace(LEFT_NAMESPACE); + util.setUpClusterWide(NAMESPACE, Set.of(LEFT_NAMESPACE)); + } + + @AfterAll + static void afterAll() throws Exception { + util.deleteNamespace(LEFT_NAMESPACE); + Commons.cleanUp(IMAGE_NAME, K3S); + } + + /** + *
+	 *     - configMap with no labels and data: left.value = left-initial exists in namespace left
+	 *     - we assert that we can read it correctly first, by invoking localhost/left
+	 *
+	 *     - then we change the configmap by adding a label, this in turn does not
+	 *       change the result of localhost/left, because the data has not changed.
+	 *
+	 *     - then we change data inside the config map, and we must see the updated value
+	 * 
+ */ + @Test + void testSimple() { + manifests(Phase.CREATE); + Commons.assertReloadLogStatements("added configmap informer for namespace", + "added secret informer for namespace", IMAGE_NAME); + + WebClient webClient = builder().baseUrl("http://localhost/" + LEFT_NAMESPACE).build(); + String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec()) + .block(); + + // we first read the initial value from the left-configmap + Assertions.assertEquals("left-initial", result); + + // then deploy a new version of left-configmap, but without changing its data, + // only add a label + V1ConfigMap configMap = new V1ConfigMapBuilder().withMetadata(new V1ObjectMetaBuilder() + .withLabels(Map.of("new-label", "abc")).withNamespace("left").withName("left-configmap").build()) + .withData(Map.of("left.value", "left-initial")).build(); + + replaceConfigMap(configMap, "left-configmap"); + + await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> { + WebClient innerWebClient = builder().baseUrl("http://localhost/" + LEFT_NAMESPACE).build(); + String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class) + .retryWhen(retrySpec()).block(); + return "left-initial".equals(innerResult); + }); + + String logs = logs(); + Assertions.assertTrue(logs.contains("ConfigMap left-configmap was updated in namespace left")); + Assertions.assertTrue(logs.contains("data in configmap has not changed, will not reload")); + + // change data + configMap = new V1ConfigMapBuilder() + .withMetadata(new V1ObjectMetaBuilder().withLabels(Map.of("new-label", "abc")).withNamespace("left") + .withName("left-configmap").build()) + .withData(Map.of("left.value", "left-after-change")).build(); + + replaceConfigMap(configMap, "left-configmap"); + + await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> { + WebClient innerWebClient = builder().baseUrl("http://localhost/" + LEFT_NAMESPACE).build(); + String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class) + .retryWhen(retrySpec()).block(); + return "left-after-change".equals(innerResult); + }); + + manifests(Phase.DELETE); + } + + private static void manifests(Phase phase) { + + try { + + V1ConfigMap leftConfigMap = (V1ConfigMap) util.yaml("left-configmap.yaml"); + + V1Deployment deployment = (V1Deployment) util.yaml("one/deployment.yaml"); + V1Service service = (V1Service) util.yaml("service.yaml"); + V1Ingress ingress = (V1Ingress) util.yaml("ingress.yaml"); + + List envVars = new ArrayList<>( + Optional.ofNullable(deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv()) + .orElse(List.of())); + + V1EnvVar secretsDisabledEnvVar = new V1EnvVar().name("SPRING_CLOUD_KUBERNETES_SECRETS_ENABLED") + .value("FALSE"); + envVars.add(secretsDisabledEnvVar); + deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars); + + if (phase.equals(Phase.CREATE)) { + util.createAndWait(LEFT_NAMESPACE, leftConfigMap, null); + util.createAndWait(NAMESPACE, null, deployment, service, ingress, true); + } + + if (phase.equals(Phase.DELETE)) { + util.deleteAndWait(LEFT_NAMESPACE, leftConfigMap, null); + util.deleteAndWait(NAMESPACE, deployment, service, ingress); + } + + } + catch (Exception e) { + throw new RuntimeException(e); + } + + } + + private String logs() { + try { + String appPodName = K3S.execInContainer("sh", "-c", + "kubectl get pods -l app=" + IMAGE_NAME + " -o=name --no-headers | tr -d '\n'").getStdout(); + + Container.ExecResult execResult = K3S.execInContainer("sh", "-c", "kubectl logs " + appPodName.trim()); + return execResult.getStdout(); + } + catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + private WebClient.Builder builder() { + return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create())); + } + + private RetryBackoffSpec retrySpec() { + return Retry.fixedDelay(120, Duration.ofSeconds(2)).filter(Objects::nonNull); + } + + private static void replaceConfigMap(V1ConfigMap configMap, String name) { + try { + api.replaceNamespacedConfigMap(name, LEFT_NAMESPACE, configMap, null, null, null, null); + } + catch (ApiException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-secrets-event-reload/src/test/java/org/springframework/cloud/kubernetes/client/secrets/event/reload/DataChangesInSecretsReloadIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-secrets-event-reload/src/test/java/org/springframework/cloud/kubernetes/client/secrets/event/reload/DataChangesInSecretsReloadIT.java new file mode 100644 index 00000000..24e5b910 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-secrets-event-reload/src/test/java/org/springframework/cloud/kubernetes/client/secrets/event/reload/DataChangesInSecretsReloadIT.java @@ -0,0 +1,212 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.secrets.event.reload; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1Deployment; +import io.kubernetes.client.openapi.models.V1EnvVar; +import io.kubernetes.client.openapi.models.V1Ingress; +import io.kubernetes.client.openapi.models.V1ObjectMetaBuilder; +import io.kubernetes.client.openapi.models.V1Secret; +import io.kubernetes.client.openapi.models.V1SecretBuilder; +import io.kubernetes.client.openapi.models.V1Service; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.Container; +import org.testcontainers.k3s.K3sContainer; +import reactor.netty.http.client.HttpClient; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; +import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; +import org.springframework.cloud.kubernetes.integration.tests.commons.native_client.Util; +import org.springframework.http.HttpMethod; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; +import org.springframework.web.reactive.function.client.WebClient; + +import static org.awaitility.Awaitility.await; + +class DataChangesInSecretsReloadIT { + + private static final String IMAGE_NAME = "spring-cloud-kubernetes-client-secrets-event-reload"; + + private static final String NAMESPACE = "default"; + + private static final K3sContainer K3S = Commons.container(); + + private static Util util; + + private static CoreV1Api api; + + @BeforeAll + static void beforeAll() throws Exception { + K3S.start(); + Commons.validateImage(IMAGE_NAME, K3S); + Commons.loadSpringCloudKubernetesImage(IMAGE_NAME, K3S); + + util = new Util(K3S); + api = new CoreV1Api(); + + util.setUpClusterWide(NAMESPACE, Set.of(NAMESPACE)); + } + + @AfterAll + static void afterAll() throws Exception { + Commons.cleanUp(IMAGE_NAME, K3S); + } + + /** + *
+	 *     - secret with no labels and data: from.properties.key = initial exists in namespace default
+	 *     - we assert that we can read it correctly first, by invoking localhost/key.
+	 *
+	 *     - then we change the secret by adding a label, this in turn does not
+	 *       change the result of localhost/key, because the data has not changed.
+	 *
+	 *     - then we change data inside the secret, and we must see the updated value.
+	 * 
+ */ + @Test + void testSimple() { + manifests(Phase.CREATE); + Commons.assertReloadLogStatements("added secret informer for namespace", + "added configmap informer for namespace", IMAGE_NAME); + + WebClient webClient = builder().baseUrl("http://localhost/key").build(); + String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec()) + .block(); + + // we first read the initial value from the secret + Assertions.assertEquals("initial", result); + + // then deploy a new version of left-configmap, but without changing its data, + // only add a label + V1Secret secret = new V1SecretBuilder() + .withMetadata(new V1ObjectMetaBuilder().withLabels(Map.of("new-label", "abc")).withNamespace(NAMESPACE) + .withName("event-reload").build()) + .withData(Map.of("application.properties", "from.properties.key=initial".getBytes())).build(); + + replaceSecret(secret, "event-reload"); + + await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> { + WebClient innerWebClient = builder().baseUrl("http://localhost/key").build(); + String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class) + .retryWhen(retrySpec()).block(); + return "initial".equals(innerResult); + }); + + String logs = logs(); + Assertions.assertTrue(logs.contains("Secret event-reload was updated in namespace default")); + Assertions.assertTrue(logs.contains("data in secret has not changed, will not reload")); + + // change data + secret = new V1SecretBuilder() + .withMetadata(new V1ObjectMetaBuilder().withLabels(Map.of("new-label", "abc")).withNamespace(NAMESPACE) + .withName("event-reload").build()) + .withData(Map.of("application.properties", "from.properties.key=change-initial".getBytes())).build(); + + replaceSecret(secret, "event-reload"); + + await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> { + WebClient innerWebClient = builder().baseUrl("http://localhost/key").build(); + String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class) + .retryWhen(retrySpec()).block(); + return "change-initial".equals(innerResult); + }); + + manifests(Phase.DELETE); + } + + private static void manifests(Phase phase) { + + try { + + V1Secret secret = (V1Secret) util.yaml("secret.yaml"); + V1Deployment deployment = (V1Deployment) util.yaml("deployment.yaml"); + V1Service service = (V1Service) util.yaml("service.yaml"); + V1Ingress ingress = (V1Ingress) util.yaml("ingress.yaml"); + + List envVars = new ArrayList<>( + Optional.ofNullable(deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv()) + .orElse(List.of())); + + V1EnvVar configDisabledEnvVar = new V1EnvVar().name("SPRING_CLOUD_KUBERNETES_CONFIG_ENABLED") + .value("FALSE"); + envVars.add(configDisabledEnvVar); + deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars); + + if (phase.equals(Phase.CREATE)) { + util.createAndWait(NAMESPACE, null, secret); + util.createAndWait(NAMESPACE, null, deployment, service, ingress, true); + } + + if (phase.equals(Phase.DELETE)) { + util.deleteAndWait(NAMESPACE, null, secret); + util.deleteAndWait(NAMESPACE, deployment, service, ingress); + } + + } + catch (Exception e) { + throw new RuntimeException(e); + } + + } + + private String logs() { + try { + String appPodName = K3S.execInContainer("sh", "-c", + "kubectl get pods -l app=" + IMAGE_NAME + " -o=name --no-headers | tr -d '\n'").getStdout(); + + Container.ExecResult execResult = K3S.execInContainer("sh", "-c", "kubectl logs " + appPodName.trim()); + return execResult.getStdout(); + } + catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + private WebClient.Builder builder() { + return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create())); + } + + private RetryBackoffSpec retrySpec() { + return Retry.fixedDelay(120, Duration.ofSeconds(2)).filter(Objects::nonNull); + } + + private static void replaceSecret(V1Secret secret, String name) { + try { + api.replaceNamespacedSecret(name, NAMESPACE, secret, null, null, null, null); + } + catch (ApiException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-secrets-event-reload/src/test/resources/secret.yaml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-secrets-event-reload/src/test/resources/secret.yaml index bf3e730d..9ad19aad 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-secrets-event-reload/src/test/resources/secret.yaml +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-secrets-event-reload/src/test/resources/secret.yaml @@ -6,4 +6,4 @@ metadata: data: # from.properties.key=initial application.properties: | - ZnJvbS5wcm9wZXJ0aWVzLmtleT1pbml0aWFsCg== + ZnJvbS5wcm9wZXJ0aWVzLmtleT1pbml0aWFs diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-configmap-event-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/configmap/event/reload/DataChangesInConfigMapReloadIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-configmap-event-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/configmap/event/reload/DataChangesInConfigMapReloadIT.java new file mode 100644 index 00000000..fe6a2d73 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-configmap-event-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/configmap/event/reload/DataChangesInConfigMapReloadIT.java @@ -0,0 +1,215 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.fabric8.configmap.event.reload; + +import java.io.InputStream; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.ConfigMapBuilder; +import io.fabric8.kubernetes.api.model.EnvVar; +import io.fabric8.kubernetes.api.model.EnvVarBuilder; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.networking.v1.Ingress; +import io.fabric8.kubernetes.client.KubernetesClient; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.Container; +import org.testcontainers.k3s.K3sContainer; +import reactor.netty.http.client.HttpClient; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; +import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; +import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util; +import org.springframework.http.HttpMethod; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; +import org.springframework.web.reactive.function.client.WebClient; + +import static org.awaitility.Awaitility.await; + +class DataChangesInConfigMapReloadIT { + + private static final String IMAGE_NAME = "spring-cloud-kubernetes-fabric8-client-configmap-event-reload"; + + private static final String NAMESPACE = "default"; + + private static final String LEFT_NAMESPACE = "left"; + + private static final K3sContainer K3S = Commons.container(); + + private static Util util; + + private static KubernetesClient client; + + @BeforeAll + static void beforeAll() throws Exception { + K3S.start(); + Commons.validateImage(IMAGE_NAME, K3S); + Commons.loadSpringCloudKubernetesImage(IMAGE_NAME, K3S); + + util = new Util(K3S); + client = util.client(); + + util.createNamespace(LEFT_NAMESPACE); + util.setUpClusterWide(NAMESPACE, Set.of(LEFT_NAMESPACE)); + } + + @AfterAll + static void afterAll() throws Exception { + util.deleteNamespace(LEFT_NAMESPACE); + Commons.cleanUp(IMAGE_NAME, K3S); + } + + /** + *
+	 *     - configMap with no labels and data: left.value = left-initial exists in namespace left
+	 *     - we assert that we can read it correctly first, by invoking localhost/left
+	 *
+	 *     - then we change the configmap by adding a label, this in turn does not
+	 *       change the result of localhost/left, because the data has not changed.
+	 *
+	 *     - then we change data inside the config map, and we must see the updated value
+	 * 
+ */ + @Test + void testSimple() { + manifests(Phase.CREATE); + Commons.assertReloadLogStatements("added configmap informer for namespace", + "added secret informer for namespace", IMAGE_NAME); + + WebClient webClient = builder().baseUrl("http://localhost/" + LEFT_NAMESPACE).build(); + String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec()) + .block(); + + // we first read the initial value from the left-configmap + Assertions.assertEquals("left-initial", result); + + // then deploy a new version of left-configmap, but without changing its data, + // only add a label + ConfigMap configMap = new ConfigMapBuilder().withMetadata(new ObjectMetaBuilder() + .withLabels(Map.of("new-label", "abc")).withNamespace("left").withName("left-configmap").build()) + .withData(Map.of("left.value", "left-initial")).build(); + + replaceConfigMap(configMap); + + await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> { + WebClient innerWebClient = builder().baseUrl("http://localhost/" + LEFT_NAMESPACE).build(); + String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class) + .retryWhen(retrySpec()).block(); + return "left-initial".equals(innerResult); + }); + + String logs = logs(); + Assertions.assertTrue(logs.contains("ConfigMap left-configmap was updated in namespace left")); + Assertions.assertTrue(logs.contains("data in configmap has not changed, will not reload")); + + // change data + configMap = new ConfigMapBuilder() + .withMetadata(new ObjectMetaBuilder().withLabels(Map.of("new-label", "abc")).withNamespace("left") + .withName("left-configmap").build()) + .withData(Map.of("left.value", "left-after-change")).build(); + + replaceConfigMap(configMap); + + await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> { + WebClient innerWebClient = builder().baseUrl("http://localhost/" + LEFT_NAMESPACE).build(); + String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class) + .retryWhen(retrySpec()).block(); + return "left-after-change".equals(innerResult); + }); + + manifests(Phase.DELETE); + } + + private static void manifests(Phase phase) { + + InputStream deploymentStream = util.inputStream("deployment.yaml"); + InputStream serviceStream = util.inputStream("service.yaml"); + InputStream ingressStream = util.inputStream("ingress.yaml"); + InputStream configmapAsStream = util.inputStream("left-configmap.yaml"); + + Deployment deployment = client.apps().deployments().load(deploymentStream).get(); + + List envVars = new ArrayList<>( + deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv()); + EnvVar activeProfileProperty = new EnvVarBuilder().withName("SPRING_PROFILES_ACTIVE").withValue("one").build(); + envVars.add(activeProfileProperty); + + EnvVar secretsDisabledEnvVar = new EnvVarBuilder().withName("SPRING_CLOUD_KUBERNETES_SECRETS_ENABLED") + .withValue("FALSE").build(); + + EnvVar debugLevel = new EnvVarBuilder() + .withName("LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_CLIENT_CONFIG_RELOAD").withName("DEBUG") + .build(); + envVars.add(debugLevel); + + envVars.add(secretsDisabledEnvVar); + deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars); + + Service service = client.services().load(serviceStream).get(); + Ingress ingress = client.network().v1().ingresses().load(ingressStream).get(); + ConfigMap configMap = client.configMaps().load(configmapAsStream).get(); + + if (phase.equals(Phase.CREATE)) { + util.createAndWait(LEFT_NAMESPACE, configMap, null); + util.createAndWait(NAMESPACE, null, deployment, service, ingress, true); + } + else { + util.deleteAndWait(LEFT_NAMESPACE, configMap, null); + util.deleteAndWait(NAMESPACE, deployment, service, ingress); + } + + } + + private String logs() { + try { + String appPodName = K3S.execInContainer("sh", "-c", + "kubectl get pods -l app=" + IMAGE_NAME + " -o=name --no-headers | tr -d '\n'").getStdout(); + + Container.ExecResult execResult = K3S.execInContainer("sh", "-c", "kubectl logs " + appPodName.trim()); + return execResult.getStdout(); + } + catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + private WebClient.Builder builder() { + return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create())); + } + + private RetryBackoffSpec retrySpec() { + return Retry.fixedDelay(120, Duration.ofSeconds(2)).filter(Objects::nonNull); + } + + private static void replaceConfigMap(ConfigMap configMap) { + client.configMaps().inNamespace(LEFT_NAMESPACE).resource(configMap).createOrReplace(); + } + +} diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-secrets-event-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/secrets/event/reload/DataChangesInSecretsReloadIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-secrets-event-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/secrets/event/reload/DataChangesInSecretsReloadIT.java new file mode 100644 index 00000000..2f81aa25 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-secrets-event-reload/src/test/java/org/springframework/cloud/kubernetes/fabric8/secrets/event/reload/DataChangesInSecretsReloadIT.java @@ -0,0 +1,208 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.fabric8.secrets.event.reload; + +import java.io.InputStream; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import io.fabric8.kubernetes.api.model.EnvVar; +import io.fabric8.kubernetes.api.model.EnvVarBuilder; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.api.model.Secret; +import io.fabric8.kubernetes.api.model.SecretBuilder; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.networking.v1.Ingress; +import io.fabric8.kubernetes.client.KubernetesClient; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.Container; +import org.testcontainers.k3s.K3sContainer; +import reactor.netty.http.client.HttpClient; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; +import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; +import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util; +import org.springframework.http.HttpMethod; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; +import org.springframework.web.reactive.function.client.WebClient; + +import static org.awaitility.Awaitility.await; + +/** + * @author wind57 + */ +class DataChangesInSecretsReloadIT { + + private static final String IMAGE_NAME = "spring-cloud-kubernetes-fabric8-client-secrets-event-reload"; + + private static final String NAMESPACE = "default"; + + private static KubernetesClient client; + + private static Util util; + + private static final K3sContainer K3S = Commons.container(); + + @BeforeAll + static void beforeAll() throws Exception { + K3S.start(); + Commons.validateImage(IMAGE_NAME, K3S); + Commons.loadSpringCloudKubernetesImage(IMAGE_NAME, K3S); + + util = new Util(K3S); + client = util.client(); + util.setUp(NAMESPACE); + } + + @AfterAll + static void after() throws Exception { + Commons.cleanUp(IMAGE_NAME, K3S); + } + + /** + *
+	 *     - secret with no labels and data: from.properties.key = initial exists in namespace default
+	 *     - we assert that we can read it correctly first, by invoking localhost/key.
+	 *
+	 *     - then we change the secret by adding a label, this in turn does not
+	 *       change the result of localhost/key, because the data has not changed.
+	 *
+	 *     - then we change data inside the secret, and we must see the updated value.
+	 * 
+ */ + @Test + void testSimple() { + manifests(Phase.CREATE); + Commons.assertReloadLogStatements("added secret informer for namespace", + "added configmap informer for namespace", IMAGE_NAME); + + WebClient webClient = builder().baseUrl("http://localhost/key").build(); + String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec()) + .block(); + Assertions.assertEquals("initial", result); + + Secret secret = new SecretBuilder() + .withMetadata(new ObjectMetaBuilder().withLabels(Map.of("letter", "a")).withNamespace(NAMESPACE) + .withName("event-reload").build()) + .withData(Map.of("application.properties", + Base64.getEncoder().encodeToString("from.properties.key=initial".getBytes()))) + .build(); + client.secrets().inNamespace(NAMESPACE).resource(secret).createOrReplace(); + + await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> { + WebClient innerWebClient = builder().baseUrl("http://localhost/key").build(); + String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class) + .retryWhen(retrySpec()).block(); + return "initial".equals(innerResult); + }); + + String logs = logs(); + Assertions.assertTrue(logs.contains("Secret event-reload was updated in namespace default")); + Assertions.assertTrue(logs.contains("data in secret has not changed, will not reload")); + + // change data + secret = new SecretBuilder() + .withMetadata(new ObjectMetaBuilder().withNamespace(NAMESPACE).withName("event-reload").build()) + .withData(Map.of("application.properties", + Base64.getEncoder().encodeToString("from.properties.key=initial-changed".getBytes()))) + .build(); + + client.secrets().inNamespace(NAMESPACE).resource(secret).createOrReplace(); + + await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> { + WebClient innerWebClient = builder().baseUrl("http://localhost/key").build(); + String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class) + .retryWhen(retrySpec()).block(); + return "initial-changed".equals(innerResult); + }); + + manifests(Phase.DELETE); + } + + private static void manifests(Phase phase) { + + InputStream deploymentStream = util.inputStream("deployment.yaml"); + InputStream serviceStream = util.inputStream("service.yaml"); + InputStream ingressStream = util.inputStream("ingress.yaml"); + InputStream secretAsStream = util.inputStream("secret.yaml"); + + Deployment deployment = client.apps().deployments().load(deploymentStream).get(); + + List envVars = new ArrayList<>( + deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv()); + EnvVar activeProfileProperty = new EnvVarBuilder().withName("SPRING_PROFILES_ACTIVE").withValue("one").build(); + envVars.add(activeProfileProperty); + + EnvVar configMapsDisabledEnvVar = new EnvVarBuilder().withName("SPRING_CLOUD_KUBERNETES_CONFIG_ENABLED") + .withValue("FALSE").build(); + + EnvVar debugLevel = new EnvVarBuilder() + .withName("LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_CLIENT_CONFIG_RELOAD").withName("DEBUG") + .build(); + envVars.add(debugLevel); + + envVars.add(configMapsDisabledEnvVar); + deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars); + + Service service = client.services().load(serviceStream).get(); + Ingress ingress = client.network().v1().ingresses().load(ingressStream).get(); + Secret secret = client.secrets().load(secretAsStream).get(); + + if (phase.equals(Phase.CREATE)) { + util.createAndWait(NAMESPACE, null, secret); + util.createAndWait(NAMESPACE, null, deployment, service, ingress, true); + } + else { + util.deleteAndWait(NAMESPACE, null, secret); + util.deleteAndWait(NAMESPACE, deployment, service, ingress); + } + + } + + private String logs() { + try { + String appPodName = K3S.execInContainer("sh", "-c", + "kubectl get pods -l app=" + IMAGE_NAME + " -o=name --no-headers | tr -d '\n'").getStdout(); + + Container.ExecResult execResult = K3S.execInContainer("sh", "-c", "kubectl logs " + appPodName.trim()); + return execResult.getStdout(); + } + catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + private WebClient.Builder builder() { + return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create())); + } + + private RetryBackoffSpec retrySpec() { + return Retry.fixedDelay(120, Duration.ofSeconds(2)).filter(Objects::nonNull); + } + +} diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-secrets-event-reload/src/test/resources/secret.yaml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-secrets-event-reload/src/test/resources/secret.yaml index bf3e730d..9ad19aad 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-secrets-event-reload/src/test/resources/secret.yaml +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-secrets-event-reload/src/test/resources/secret.yaml @@ -6,4 +6,4 @@ metadata: data: # from.properties.key=initial application.properties: | - ZnJvbS5wcm9wZXJ0aWVzLmtleT1pbml0aWFsCg== + ZnJvbS5wcm9wZXJ0aWVzLmtleT1pbml0aWFs