Fix 1323 part 2 (#1329)

This commit is contained in:
erabii
2023-04-28 16:56:41 +03:00
committed by GitHub
parent cd24fda5bc
commit 7fa07e66d7
12 changed files with 1049 additions and 50 deletions

View File

@@ -18,6 +18,7 @@ package org.springframework.cloud.kubernetes.client.config.reload;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import io.kubernetes.client.common.KubernetesObject;
@@ -81,7 +82,12 @@ public class KubernetesClientEventBasedConfigMapChangeDetector extends Configura
public void onUpdate(V1ConfigMap oldConfigMap, V1ConfigMap newConfigMap) {
LOG.debug(() -> "ConfigMap " + newConfigMap.getMetadata().getName() + " was updated in namespace "
+ newConfigMap.getMetadata().getNamespace());
onEvent(newConfigMap);
if (Objects.equals(oldConfigMap.getData(), newConfigMap.getData())) {
LOG.debug(() -> "data in configmap has not changed, will not reload");
}
else {
onEvent(newConfigMap);
}
}
@Override

View File

@@ -17,7 +17,10 @@
package org.springframework.cloud.kubernetes.client.config.reload;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import io.kubernetes.client.common.KubernetesObject;
@@ -81,7 +84,13 @@ public class KubernetesClientEventBasedSecretsChangeDetector extends Configurati
public void onUpdate(V1Secret oldSecret, V1Secret newSecret) {
LOG.debug(() -> "Secret " + newSecret.getMetadata().getName() + " was updated in namespace "
+ newSecret.getMetadata().getNamespace());
onEvent(newSecret);
if (KubernetesClientEventBasedSecretsChangeDetector.equals(oldSecret.getData(), newSecret.getData())) {
LOG.debug(() -> "data in secret has not changed, will not reload");
}
else {
onEvent(newSecret);
}
}
@Override
@@ -146,4 +155,22 @@ public class KubernetesClientEventBasedSecretsChangeDetector extends Configurati
}
}
static boolean equals(Map<String, byte[]> left, Map<String, byte[]> right) {
Map<String, byte[]> innerLeft = Optional.ofNullable(left).orElse(Map.of());
Map<String, byte[]> innerRight = Optional.ofNullable(right).orElse(Map.of());
if (innerLeft.size() != innerRight.size()) {
return false;
}
for (Map.Entry<String, byte[]> entry : innerLeft.entrySet()) {
String key = entry.getKey();
byte[] value = entry.getValue();
if (!Arrays.equals(value, innerRight.get(key))) {
return false;
}
}
return true;
}
}

View File

@@ -16,22 +16,16 @@
package org.springframework.cloud.kubernetes.client.config.reload;
import java.io.IOException;
import java.lang.reflect.Modifier;
import java.time.Duration;
import java.time.OffsetDateTime;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import com.github.tomakehurst.wiremock.WireMockServer;
import com.github.tomakehurst.wiremock.client.WireMock;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.TypeAdapter;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import com.github.tomakehurst.wiremock.matching.StringValuePattern;
import io.kubernetes.client.informer.EventType;
import io.kubernetes.client.openapi.ApiClient;
import io.kubernetes.client.openapi.JSON;
@@ -45,6 +39,7 @@ import io.kubernetes.client.util.Watch;
import okhttp3.OkHttpClient;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@ -71,6 +66,12 @@ import static org.mockito.Mockito.when;
*/
class KubernetesClientEventBasedSecretsChangeDetectorTests {
private static final Map<String, StringValuePattern> WATCH_FALSE = Map.of("watch", equalTo("false"));
private static final Map<String, StringValuePattern> WATCH_TRUE = Map.of("watch", equalTo("true"));
private static final String SCENARIO = "watch";
private static WireMockServer wireMockServer;
@BeforeAll
@@ -93,43 +94,45 @@ class KubernetesClientEventBasedSecretsChangeDetectorTests {
@Test
void watch() {
GsonBuilder builder = new GsonBuilder();
builder.excludeFieldsWithModifiers(Modifier.STATIC, Modifier.TRANSIENT, Modifier.VOLATILE)
.registerTypeAdapter(OffsetDateTime.class, new GsonOffsetDateTimeAdapter());
Gson gson = builder.create();
V1Secret dbPassword = new V1Secret().kind("Secret").metadata(new V1ObjectMeta().name("db-password"))
V1Secret dbPassword = new V1Secret().metadata(new V1ObjectMeta().name("db-password"))
.putStringDataItem("password", Base64.getEncoder().encodeToString("p455w0rd".getBytes()))
.putStringDataItem("username", Base64.getEncoder().encodeToString("user".getBytes()));
V1Secret dbPasswordUpdated = new V1Secret().kind("Secret").metadata(new V1ObjectMeta().name("db-password"))
.putStringDataItem("password", Base64.getEncoder().encodeToString("p455w0rd2".getBytes()))
.putStringDataItem("username", Base64.getEncoder().encodeToString("user".getBytes()));
V1SecretList secretList = new V1SecretList().kind("SecretList").metadata(new V1ListMeta().resourceVersion("0"))
.putDataItem("password", Base64.getEncoder().encode("p455w0rd".getBytes()))
.putStringDataItem("username", Base64.getEncoder().encodeToString("user".getBytes()))
.putDataItem("username", Base64.getEncoder().encode("user".getBytes()));
V1SecretList secretList = new V1SecretList().metadata(new V1ListMeta().resourceVersion("0"))
.items(List.of(dbPassword));
stubFor(get(urlMatching("^/api/v1/namespaces/default/secrets.*")).inScenario("watch")
.whenScenarioStateIs(STARTED).withQueryParam("watch", equalTo("false"))
.willReturn(aResponse().withStatus(200).withBody(gson.toJson(secretList))).willSetStateTo("update"));
V1Secret dbPasswordUpdated = new V1Secret().metadata(new V1ObjectMeta().name("db-password"))
.putStringDataItem("password", Base64.getEncoder().encodeToString("p455w0rd2".getBytes()))
.putDataItem("password", Base64.getEncoder().encode("p455w0rd2".getBytes()))
.putStringDataItem("username", Base64.getEncoder().encodeToString("user".getBytes()))
.putDataItem("username", Base64.getEncoder().encode("user".getBytes()));
Watch.Response<V1Secret> watchResponse = new Watch.Response<>(EventType.MODIFIED.name(), dbPasswordUpdated);
stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).inScenario("watch")
.whenScenarioStateIs("update").withQueryParam("watch", equalTo("true"))
stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).inScenario(SCENARIO)
.whenScenarioStateIs(STARTED).withQueryParams(WATCH_FALSE)
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(secretList)))
.willSetStateTo("update"));
stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).inScenario(SCENARIO)
.whenScenarioStateIs("update").withQueryParams(WATCH_TRUE)
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(watchResponse)))
.willSetStateTo("add"));
stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).inScenario("watch").whenScenarioStateIs("add")
.withQueryParam("watch", equalTo("true"))
stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).inScenario(SCENARIO).whenScenarioStateIs("add")
.withQueryParams(WATCH_TRUE)
.willReturn(aResponse().withStatus(200)
.withBody(new JSON().serialize(new Watch.Response<>(EventType.ADDED.name(),
new V1Secret().kind("Secret").metadata(new V1ObjectMeta().name("rabbit-password"))
new V1Secret().metadata(new V1ObjectMeta().name("rabbit-password"))
.putDataItem("rabbit-pw", Base64.getEncoder().encode("password".getBytes()))))))
.willSetStateTo("delete"));
stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).inScenario("watch")
.whenScenarioStateIs("delete").withQueryParam("watch", equalTo("true"))
stubFor(get(urlMatching("/api/v1/namespaces/default/secrets.*")).inScenario(SCENARIO)
.whenScenarioStateIs("delete").withQueryParams(WATCH_TRUE)
.willReturn(aResponse().withStatus(200)
.withBody(new JSON().serialize(new Watch.Response<>(EventType.DELETED.name(),
new V1Secret().kind("Secret").metadata(new V1ObjectMeta().name("rabbit-password"))
new V1Secret().metadata(new V1ObjectMeta().name("rabbit-password"))
.putDataItem("rabbit-pw", Base64.getEncoder().encode("password".getBytes()))))))
.willSetStateTo("done"));
@@ -167,21 +170,121 @@ class KubernetesClientEventBasedSecretsChangeDetectorTests {
await().timeout(Duration.ofSeconds(10)).pollInterval(Duration.ofSeconds(2)).until(() -> howMany[0] >= 4);
}
// This is needed when using JDK17 because GSON uses reflection to construct an
// OffsetDateTime but that constructor
// is protected.
public final static class GsonOffsetDateTimeAdapter extends TypeAdapter<OffsetDateTime> {
/**
* both are null, treat that as no change.
*/
@Test
void equalsOne() {
Map<String, byte[]> left = null;
Map<String, byte[]> right = null;
@Override
public void write(JsonWriter jsonWriter, OffsetDateTime localDateTime) throws IOException {
jsonWriter.value(OffsetDateTime.now().toString());
}
boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right);
Assertions.assertTrue(result);
}
@Override
public OffsetDateTime read(JsonReader jsonReader) {
return OffsetDateTime.now();
}
/**
* - left is empty map
* - right is null
*
* treat as equal, that is: no change
*/
@Test
void equalsTwo() {
Map<String, byte[]> left = Map.of();
Map<String, byte[]> right = null;
boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right);
Assertions.assertTrue(result);
}
/**
* - left is empty map
* - right is null
*
* treat as equal, that is: no change
*/
@Test
void equalsThree() {
Map<String, byte[]> left = Map.of();
Map<String, byte[]> right = null;
boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right);
Assertions.assertTrue(result);
}
/**
* - left is null
* - right is empty map
*
* treat as equal, that is: no change
*/
@Test
void equalsFour() {
Map<String, byte[]> left = null;
Map<String, byte[]> right = Map.of();
boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right);
Assertions.assertTrue(result);
}
/**
* - left is empty map
* - right is empty map
*
* treat as equal, that is: no change
*/
@Test
void equalsFive() {
Map<String, byte[]> left = Map.of();
Map<String, byte[]> right = Map.of();
boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right);
Assertions.assertTrue(result);
}
/**
* - left is empty map
* - right is [1, b]
*
* treat as non-equal, that is change
*/
@Test
void equalsSix() {
Map<String, byte[]> left = Map.of();
Map<String, byte[]> right = Map.of("1", "b".getBytes());
boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right);
Assertions.assertFalse(result);
}
/**
* - left is [1, a]
* - right is [1, b]
*
* treat as non-equal, that is change
*/
@Test
void equalsSeven() {
Map<String, byte[]> left = Map.of("1", "a".getBytes());
Map<String, byte[]> right = Map.of("1", "b".getBytes());
boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right);
Assertions.assertFalse(result);
}
/**
* - left is [1, a, 2 aa]
* - right is [1, b, 2, aa]
*
* treat as non-equal, that is change
*/
@Test
void equalsEight() {
Map<String, byte[]> left = Map.of("1", "a".getBytes(), "2", "aa".getBytes());
Map<String, byte[]> right = Map.of("1", "b".getBytes(), "2", "aa".getBytes());
boolean result = KubernetesClientEventBasedSecretsChangeDetector.equals(left, right);
Assertions.assertFalse(result);
}
}

View File

@@ -52,11 +52,11 @@ public final class ConfigReloadUtil {
boolean changed = changed(sourceFromK8s, existingSources);
if (changed) {
LOG.info("Detected change in config maps");
LOG.info("Detected change in config maps/secrets");
return true;
}
else {
LOG.debug("No change detected in config maps, reload will not happen");
LOG.debug("No change detected in config maps/secrets, reload will not happen");
}
return false;

View File

@@ -19,6 +19,7 @@ package org.springframework.cloud.kubernetes.fabric8.config.reload;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import io.fabric8.kubernetes.api.model.ConfigMap;
@@ -131,7 +132,12 @@ public class Fabric8EventBasedConfigMapChangeDetector extends ConfigurationChang
public void onUpdate(ConfigMap oldConfigMap, ConfigMap newConfigMap) {
LOG.debug("ConfigMap " + newConfigMap.getMetadata().getName() + " was updated in namespace "
+ newConfigMap.getMetadata().getNamespace());
onEvent(newConfigMap);
if (Objects.equals(oldConfigMap.getData(), newConfigMap.getData())) {
LOG.debug(() -> "data in configmap has not changed, will not reload");
}
else {
onEvent(newConfigMap);
}
}
@Override

View File

@@ -19,6 +19,7 @@ package org.springframework.cloud.kubernetes.fabric8.config.reload;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import io.fabric8.kubernetes.api.model.Secret;
@@ -133,7 +134,12 @@ public class Fabric8EventBasedSecretsChangeDetector extends ConfigurationChangeD
public void onUpdate(Secret oldSecret, Secret newSecret) {
LOG.debug("Secret " + newSecret.getMetadata().getName() + " was updated in namespace "
+ newSecret.getMetadata().getNamespace());
onEvent(newSecret);
if (Objects.equals(oldSecret.getData(), newSecret.getData())) {
LOG.debug(() -> "data in secret has not changed, will not reload");
}
else {
onEvent(newSecret);
}
}
@Override

View File

@@ -0,0 +1,216 @@
/*
* Copyright 2013-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.configmap.event.reload;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.models.V1ConfigMap;
import io.kubernetes.client.openapi.models.V1ConfigMapBuilder;
import io.kubernetes.client.openapi.models.V1Deployment;
import io.kubernetes.client.openapi.models.V1EnvVar;
import io.kubernetes.client.openapi.models.V1Ingress;
import io.kubernetes.client.openapi.models.V1ObjectMetaBuilder;
import io.kubernetes.client.openapi.models.V1Service;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.Container;
import org.testcontainers.k3s.K3sContainer;
import reactor.netty.http.client.HttpClient;
import reactor.util.retry.Retry;
import reactor.util.retry.RetryBackoffSpec;
import org.springframework.cloud.kubernetes.integration.tests.commons.Commons;
import org.springframework.cloud.kubernetes.integration.tests.commons.Phase;
import org.springframework.cloud.kubernetes.integration.tests.commons.native_client.Util;
import org.springframework.http.HttpMethod;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.web.reactive.function.client.WebClient;
import static org.awaitility.Awaitility.await;
class DataChangesInConfigMapReloadIT {
private static final String IMAGE_NAME = "spring-cloud-kubernetes-client-configmap-event-reload";
private static final String NAMESPACE = "default";
private static final String LEFT_NAMESPACE = "left";
private static final K3sContainer K3S = Commons.container();
private static Util util;
private static CoreV1Api api;
@BeforeAll
static void beforeAll() throws Exception {
K3S.start();
Commons.validateImage(IMAGE_NAME, K3S);
Commons.loadSpringCloudKubernetesImage(IMAGE_NAME, K3S);
util = new Util(K3S);
api = new CoreV1Api();
util.createNamespace(LEFT_NAMESPACE);
util.setUpClusterWide(NAMESPACE, Set.of(LEFT_NAMESPACE));
}
@AfterAll
static void afterAll() throws Exception {
util.deleteNamespace(LEFT_NAMESPACE);
Commons.cleanUp(IMAGE_NAME, K3S);
}
/**
* <pre>
* - configMap with no labels and data: left.value = left-initial exists in namespace left
* - we assert that we can read it correctly first, by invoking localhost/left
*
* - then we change the configmap by adding a label, this in turn does not
* change the result of localhost/left, because the data has not changed.
*
* - then we change data inside the config map, and we must see the updated value
* </pre>
*/
@Test
void testSimple() {
manifests(Phase.CREATE);
Commons.assertReloadLogStatements("added configmap informer for namespace",
"added secret informer for namespace", IMAGE_NAME);
WebClient webClient = builder().baseUrl("http://localhost/" + LEFT_NAMESPACE).build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec())
.block();
// we first read the initial value from the left-configmap
Assertions.assertEquals("left-initial", result);
// then deploy a new version of left-configmap, but without changing its data,
// only add a label
V1ConfigMap configMap = new V1ConfigMapBuilder().withMetadata(new V1ObjectMetaBuilder()
.withLabels(Map.of("new-label", "abc")).withNamespace("left").withName("left-configmap").build())
.withData(Map.of("left.value", "left-initial")).build();
replaceConfigMap(configMap, "left-configmap");
await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> {
WebClient innerWebClient = builder().baseUrl("http://localhost/" + LEFT_NAMESPACE).build();
String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
.retryWhen(retrySpec()).block();
return "left-initial".equals(innerResult);
});
String logs = logs();
Assertions.assertTrue(logs.contains("ConfigMap left-configmap was updated in namespace left"));
Assertions.assertTrue(logs.contains("data in configmap has not changed, will not reload"));
// change data
configMap = new V1ConfigMapBuilder()
.withMetadata(new V1ObjectMetaBuilder().withLabels(Map.of("new-label", "abc")).withNamespace("left")
.withName("left-configmap").build())
.withData(Map.of("left.value", "left-after-change")).build();
replaceConfigMap(configMap, "left-configmap");
await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> {
WebClient innerWebClient = builder().baseUrl("http://localhost/" + LEFT_NAMESPACE).build();
String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
.retryWhen(retrySpec()).block();
return "left-after-change".equals(innerResult);
});
manifests(Phase.DELETE);
}
private static void manifests(Phase phase) {
try {
V1ConfigMap leftConfigMap = (V1ConfigMap) util.yaml("left-configmap.yaml");
V1Deployment deployment = (V1Deployment) util.yaml("one/deployment.yaml");
V1Service service = (V1Service) util.yaml("service.yaml");
V1Ingress ingress = (V1Ingress) util.yaml("ingress.yaml");
List<V1EnvVar> envVars = new ArrayList<>(
Optional.ofNullable(deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv())
.orElse(List.of()));
V1EnvVar secretsDisabledEnvVar = new V1EnvVar().name("SPRING_CLOUD_KUBERNETES_SECRETS_ENABLED")
.value("FALSE");
envVars.add(secretsDisabledEnvVar);
deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars);
if (phase.equals(Phase.CREATE)) {
util.createAndWait(LEFT_NAMESPACE, leftConfigMap, null);
util.createAndWait(NAMESPACE, null, deployment, service, ingress, true);
}
if (phase.equals(Phase.DELETE)) {
util.deleteAndWait(LEFT_NAMESPACE, leftConfigMap, null);
util.deleteAndWait(NAMESPACE, deployment, service, ingress);
}
}
catch (Exception e) {
throw new RuntimeException(e);
}
}
private String logs() {
try {
String appPodName = K3S.execInContainer("sh", "-c",
"kubectl get pods -l app=" + IMAGE_NAME + " -o=name --no-headers | tr -d '\n'").getStdout();
Container.ExecResult execResult = K3S.execInContainer("sh", "-c", "kubectl logs " + appPodName.trim());
return execResult.getStdout();
}
catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
private WebClient.Builder builder() {
return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create()));
}
private RetryBackoffSpec retrySpec() {
return Retry.fixedDelay(120, Duration.ofSeconds(2)).filter(Objects::nonNull);
}
private static void replaceConfigMap(V1ConfigMap configMap, String name) {
try {
api.replaceNamespacedConfigMap(name, LEFT_NAMESPACE, configMap, null, null, null, null);
}
catch (ApiException e) {
throw new RuntimeException(e);
}
}
}

View File

@@ -0,0 +1,212 @@
/*
* Copyright 2013-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.secrets.event.reload;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.models.V1Deployment;
import io.kubernetes.client.openapi.models.V1EnvVar;
import io.kubernetes.client.openapi.models.V1Ingress;
import io.kubernetes.client.openapi.models.V1ObjectMetaBuilder;
import io.kubernetes.client.openapi.models.V1Secret;
import io.kubernetes.client.openapi.models.V1SecretBuilder;
import io.kubernetes.client.openapi.models.V1Service;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.Container;
import org.testcontainers.k3s.K3sContainer;
import reactor.netty.http.client.HttpClient;
import reactor.util.retry.Retry;
import reactor.util.retry.RetryBackoffSpec;
import org.springframework.cloud.kubernetes.integration.tests.commons.Commons;
import org.springframework.cloud.kubernetes.integration.tests.commons.Phase;
import org.springframework.cloud.kubernetes.integration.tests.commons.native_client.Util;
import org.springframework.http.HttpMethod;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.web.reactive.function.client.WebClient;
import static org.awaitility.Awaitility.await;
class DataChangesInSecretsReloadIT {
private static final String IMAGE_NAME = "spring-cloud-kubernetes-client-secrets-event-reload";
private static final String NAMESPACE = "default";
private static final K3sContainer K3S = Commons.container();
private static Util util;
private static CoreV1Api api;
@BeforeAll
static void beforeAll() throws Exception {
K3S.start();
Commons.validateImage(IMAGE_NAME, K3S);
Commons.loadSpringCloudKubernetesImage(IMAGE_NAME, K3S);
util = new Util(K3S);
api = new CoreV1Api();
util.setUpClusterWide(NAMESPACE, Set.of(NAMESPACE));
}
@AfterAll
static void afterAll() throws Exception {
Commons.cleanUp(IMAGE_NAME, K3S);
}
/**
* <pre>
* - secret with no labels and data: from.properties.key = initial exists in namespace default
* - we assert that we can read it correctly first, by invoking localhost/key.
*
* - then we change the secret by adding a label, this in turn does not
* change the result of localhost/key, because the data has not changed.
*
* - then we change data inside the secret, and we must see the updated value.
* </pre>
*/
@Test
void testSimple() {
manifests(Phase.CREATE);
Commons.assertReloadLogStatements("added secret informer for namespace",
"added configmap informer for namespace", IMAGE_NAME);
WebClient webClient = builder().baseUrl("http://localhost/key").build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec())
.block();
// we first read the initial value from the secret
Assertions.assertEquals("initial", result);
// then deploy a new version of left-configmap, but without changing its data,
// only add a label
V1Secret secret = new V1SecretBuilder()
.withMetadata(new V1ObjectMetaBuilder().withLabels(Map.of("new-label", "abc")).withNamespace(NAMESPACE)
.withName("event-reload").build())
.withData(Map.of("application.properties", "from.properties.key=initial".getBytes())).build();
replaceSecret(secret, "event-reload");
await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> {
WebClient innerWebClient = builder().baseUrl("http://localhost/key").build();
String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
.retryWhen(retrySpec()).block();
return "initial".equals(innerResult);
});
String logs = logs();
Assertions.assertTrue(logs.contains("Secret event-reload was updated in namespace default"));
Assertions.assertTrue(logs.contains("data in secret has not changed, will not reload"));
// change data
secret = new V1SecretBuilder()
.withMetadata(new V1ObjectMetaBuilder().withLabels(Map.of("new-label", "abc")).withNamespace(NAMESPACE)
.withName("event-reload").build())
.withData(Map.of("application.properties", "from.properties.key=change-initial".getBytes())).build();
replaceSecret(secret, "event-reload");
await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> {
WebClient innerWebClient = builder().baseUrl("http://localhost/key").build();
String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
.retryWhen(retrySpec()).block();
return "change-initial".equals(innerResult);
});
manifests(Phase.DELETE);
}
private static void manifests(Phase phase) {
try {
V1Secret secret = (V1Secret) util.yaml("secret.yaml");
V1Deployment deployment = (V1Deployment) util.yaml("deployment.yaml");
V1Service service = (V1Service) util.yaml("service.yaml");
V1Ingress ingress = (V1Ingress) util.yaml("ingress.yaml");
List<V1EnvVar> envVars = new ArrayList<>(
Optional.ofNullable(deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv())
.orElse(List.of()));
V1EnvVar configDisabledEnvVar = new V1EnvVar().name("SPRING_CLOUD_KUBERNETES_CONFIG_ENABLED")
.value("FALSE");
envVars.add(configDisabledEnvVar);
deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars);
if (phase.equals(Phase.CREATE)) {
util.createAndWait(NAMESPACE, null, secret);
util.createAndWait(NAMESPACE, null, deployment, service, ingress, true);
}
if (phase.equals(Phase.DELETE)) {
util.deleteAndWait(NAMESPACE, null, secret);
util.deleteAndWait(NAMESPACE, deployment, service, ingress);
}
}
catch (Exception e) {
throw new RuntimeException(e);
}
}
private String logs() {
try {
String appPodName = K3S.execInContainer("sh", "-c",
"kubectl get pods -l app=" + IMAGE_NAME + " -o=name --no-headers | tr -d '\n'").getStdout();
Container.ExecResult execResult = K3S.execInContainer("sh", "-c", "kubectl logs " + appPodName.trim());
return execResult.getStdout();
}
catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
private WebClient.Builder builder() {
return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create()));
}
private RetryBackoffSpec retrySpec() {
return Retry.fixedDelay(120, Duration.ofSeconds(2)).filter(Objects::nonNull);
}
private static void replaceSecret(V1Secret secret, String name) {
try {
api.replaceNamespacedSecret(name, NAMESPACE, secret, null, null, null, null);
}
catch (ApiException e) {
throw new RuntimeException(e);
}
}
}

View File

@@ -6,4 +6,4 @@ metadata:
data:
# from.properties.key=initial
application.properties: |
ZnJvbS5wcm9wZXJ0aWVzLmtleT1pbml0aWFsCg==
ZnJvbS5wcm9wZXJ0aWVzLmtleT1pbml0aWFs

View File

@@ -0,0 +1,215 @@
/*
* Copyright 2013-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.configmap.event.reload;
import java.io.InputStream;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.api.model.ConfigMapBuilder;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.EnvVarBuilder;
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.networking.v1.Ingress;
import io.fabric8.kubernetes.client.KubernetesClient;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.Container;
import org.testcontainers.k3s.K3sContainer;
import reactor.netty.http.client.HttpClient;
import reactor.util.retry.Retry;
import reactor.util.retry.RetryBackoffSpec;
import org.springframework.cloud.kubernetes.integration.tests.commons.Commons;
import org.springframework.cloud.kubernetes.integration.tests.commons.Phase;
import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util;
import org.springframework.http.HttpMethod;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.web.reactive.function.client.WebClient;
import static org.awaitility.Awaitility.await;
class DataChangesInConfigMapReloadIT {
private static final String IMAGE_NAME = "spring-cloud-kubernetes-fabric8-client-configmap-event-reload";
private static final String NAMESPACE = "default";
private static final String LEFT_NAMESPACE = "left";
private static final K3sContainer K3S = Commons.container();
private static Util util;
private static KubernetesClient client;
@BeforeAll
static void beforeAll() throws Exception {
K3S.start();
Commons.validateImage(IMAGE_NAME, K3S);
Commons.loadSpringCloudKubernetesImage(IMAGE_NAME, K3S);
util = new Util(K3S);
client = util.client();
util.createNamespace(LEFT_NAMESPACE);
util.setUpClusterWide(NAMESPACE, Set.of(LEFT_NAMESPACE));
}
@AfterAll
static void afterAll() throws Exception {
util.deleteNamespace(LEFT_NAMESPACE);
Commons.cleanUp(IMAGE_NAME, K3S);
}
/**
* <pre>
* - configMap with no labels and data: left.value = left-initial exists in namespace left
* - we assert that we can read it correctly first, by invoking localhost/left
*
* - then we change the configmap by adding a label, this in turn does not
* change the result of localhost/left, because the data has not changed.
*
* - then we change data inside the config map, and we must see the updated value
* </pre>
*/
@Test
void testSimple() {
manifests(Phase.CREATE);
Commons.assertReloadLogStatements("added configmap informer for namespace",
"added secret informer for namespace", IMAGE_NAME);
WebClient webClient = builder().baseUrl("http://localhost/" + LEFT_NAMESPACE).build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec())
.block();
// we first read the initial value from the left-configmap
Assertions.assertEquals("left-initial", result);
// then deploy a new version of left-configmap, but without changing its data,
// only add a label
ConfigMap configMap = new ConfigMapBuilder().withMetadata(new ObjectMetaBuilder()
.withLabels(Map.of("new-label", "abc")).withNamespace("left").withName("left-configmap").build())
.withData(Map.of("left.value", "left-initial")).build();
replaceConfigMap(configMap);
await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> {
WebClient innerWebClient = builder().baseUrl("http://localhost/" + LEFT_NAMESPACE).build();
String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
.retryWhen(retrySpec()).block();
return "left-initial".equals(innerResult);
});
String logs = logs();
Assertions.assertTrue(logs.contains("ConfigMap left-configmap was updated in namespace left"));
Assertions.assertTrue(logs.contains("data in configmap has not changed, will not reload"));
// change data
configMap = new ConfigMapBuilder()
.withMetadata(new ObjectMetaBuilder().withLabels(Map.of("new-label", "abc")).withNamespace("left")
.withName("left-configmap").build())
.withData(Map.of("left.value", "left-after-change")).build();
replaceConfigMap(configMap);
await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> {
WebClient innerWebClient = builder().baseUrl("http://localhost/" + LEFT_NAMESPACE).build();
String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
.retryWhen(retrySpec()).block();
return "left-after-change".equals(innerResult);
});
manifests(Phase.DELETE);
}
private static void manifests(Phase phase) {
InputStream deploymentStream = util.inputStream("deployment.yaml");
InputStream serviceStream = util.inputStream("service.yaml");
InputStream ingressStream = util.inputStream("ingress.yaml");
InputStream configmapAsStream = util.inputStream("left-configmap.yaml");
Deployment deployment = client.apps().deployments().load(deploymentStream).get();
List<EnvVar> envVars = new ArrayList<>(
deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv());
EnvVar activeProfileProperty = new EnvVarBuilder().withName("SPRING_PROFILES_ACTIVE").withValue("one").build();
envVars.add(activeProfileProperty);
EnvVar secretsDisabledEnvVar = new EnvVarBuilder().withName("SPRING_CLOUD_KUBERNETES_SECRETS_ENABLED")
.withValue("FALSE").build();
EnvVar debugLevel = new EnvVarBuilder()
.withName("LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_CLIENT_CONFIG_RELOAD").withName("DEBUG")
.build();
envVars.add(debugLevel);
envVars.add(secretsDisabledEnvVar);
deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars);
Service service = client.services().load(serviceStream).get();
Ingress ingress = client.network().v1().ingresses().load(ingressStream).get();
ConfigMap configMap = client.configMaps().load(configmapAsStream).get();
if (phase.equals(Phase.CREATE)) {
util.createAndWait(LEFT_NAMESPACE, configMap, null);
util.createAndWait(NAMESPACE, null, deployment, service, ingress, true);
}
else {
util.deleteAndWait(LEFT_NAMESPACE, configMap, null);
util.deleteAndWait(NAMESPACE, deployment, service, ingress);
}
}
private String logs() {
try {
String appPodName = K3S.execInContainer("sh", "-c",
"kubectl get pods -l app=" + IMAGE_NAME + " -o=name --no-headers | tr -d '\n'").getStdout();
Container.ExecResult execResult = K3S.execInContainer("sh", "-c", "kubectl logs " + appPodName.trim());
return execResult.getStdout();
}
catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
private WebClient.Builder builder() {
return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create()));
}
private RetryBackoffSpec retrySpec() {
return Retry.fixedDelay(120, Duration.ofSeconds(2)).filter(Objects::nonNull);
}
private static void replaceConfigMap(ConfigMap configMap) {
client.configMaps().inNamespace(LEFT_NAMESPACE).resource(configMap).createOrReplace();
}
}

View File

@@ -0,0 +1,208 @@
/*
* Copyright 2013-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.secrets.event.reload;
import java.io.InputStream;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.EnvVarBuilder;
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
import io.fabric8.kubernetes.api.model.Secret;
import io.fabric8.kubernetes.api.model.SecretBuilder;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.networking.v1.Ingress;
import io.fabric8.kubernetes.client.KubernetesClient;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.Container;
import org.testcontainers.k3s.K3sContainer;
import reactor.netty.http.client.HttpClient;
import reactor.util.retry.Retry;
import reactor.util.retry.RetryBackoffSpec;
import org.springframework.cloud.kubernetes.integration.tests.commons.Commons;
import org.springframework.cloud.kubernetes.integration.tests.commons.Phase;
import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util;
import org.springframework.http.HttpMethod;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.web.reactive.function.client.WebClient;
import static org.awaitility.Awaitility.await;
/**
* @author wind57
*/
class DataChangesInSecretsReloadIT {
private static final String IMAGE_NAME = "spring-cloud-kubernetes-fabric8-client-secrets-event-reload";
private static final String NAMESPACE = "default";
private static KubernetesClient client;
private static Util util;
private static final K3sContainer K3S = Commons.container();
@BeforeAll
static void beforeAll() throws Exception {
K3S.start();
Commons.validateImage(IMAGE_NAME, K3S);
Commons.loadSpringCloudKubernetesImage(IMAGE_NAME, K3S);
util = new Util(K3S);
client = util.client();
util.setUp(NAMESPACE);
}
@AfterAll
static void after() throws Exception {
Commons.cleanUp(IMAGE_NAME, K3S);
}
/**
* <pre>
* - secret with no labels and data: from.properties.key = initial exists in namespace default
* - we assert that we can read it correctly first, by invoking localhost/key.
*
* - then we change the secret by adding a label, this in turn does not
* change the result of localhost/key, because the data has not changed.
*
* - then we change data inside the secret, and we must see the updated value.
* </pre>
*/
@Test
void testSimple() {
manifests(Phase.CREATE);
Commons.assertReloadLogStatements("added secret informer for namespace",
"added configmap informer for namespace", IMAGE_NAME);
WebClient webClient = builder().baseUrl("http://localhost/key").build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec())
.block();
Assertions.assertEquals("initial", result);
Secret secret = new SecretBuilder()
.withMetadata(new ObjectMetaBuilder().withLabels(Map.of("letter", "a")).withNamespace(NAMESPACE)
.withName("event-reload").build())
.withData(Map.of("application.properties",
Base64.getEncoder().encodeToString("from.properties.key=initial".getBytes())))
.build();
client.secrets().inNamespace(NAMESPACE).resource(secret).createOrReplace();
await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> {
WebClient innerWebClient = builder().baseUrl("http://localhost/key").build();
String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
.retryWhen(retrySpec()).block();
return "initial".equals(innerResult);
});
String logs = logs();
Assertions.assertTrue(logs.contains("Secret event-reload was updated in namespace default"));
Assertions.assertTrue(logs.contains("data in secret has not changed, will not reload"));
// change data
secret = new SecretBuilder()
.withMetadata(new ObjectMetaBuilder().withNamespace(NAMESPACE).withName("event-reload").build())
.withData(Map.of("application.properties",
Base64.getEncoder().encodeToString("from.properties.key=initial-changed".getBytes())))
.build();
client.secrets().inNamespace(NAMESPACE).resource(secret).createOrReplace();
await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> {
WebClient innerWebClient = builder().baseUrl("http://localhost/key").build();
String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
.retryWhen(retrySpec()).block();
return "initial-changed".equals(innerResult);
});
manifests(Phase.DELETE);
}
private static void manifests(Phase phase) {
InputStream deploymentStream = util.inputStream("deployment.yaml");
InputStream serviceStream = util.inputStream("service.yaml");
InputStream ingressStream = util.inputStream("ingress.yaml");
InputStream secretAsStream = util.inputStream("secret.yaml");
Deployment deployment = client.apps().deployments().load(deploymentStream).get();
List<EnvVar> envVars = new ArrayList<>(
deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv());
EnvVar activeProfileProperty = new EnvVarBuilder().withName("SPRING_PROFILES_ACTIVE").withValue("one").build();
envVars.add(activeProfileProperty);
EnvVar configMapsDisabledEnvVar = new EnvVarBuilder().withName("SPRING_CLOUD_KUBERNETES_CONFIG_ENABLED")
.withValue("FALSE").build();
EnvVar debugLevel = new EnvVarBuilder()
.withName("LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_CLIENT_CONFIG_RELOAD").withName("DEBUG")
.build();
envVars.add(debugLevel);
envVars.add(configMapsDisabledEnvVar);
deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars);
Service service = client.services().load(serviceStream).get();
Ingress ingress = client.network().v1().ingresses().load(ingressStream).get();
Secret secret = client.secrets().load(secretAsStream).get();
if (phase.equals(Phase.CREATE)) {
util.createAndWait(NAMESPACE, null, secret);
util.createAndWait(NAMESPACE, null, deployment, service, ingress, true);
}
else {
util.deleteAndWait(NAMESPACE, null, secret);
util.deleteAndWait(NAMESPACE, deployment, service, ingress);
}
}
private String logs() {
try {
String appPodName = K3S.execInContainer("sh", "-c",
"kubectl get pods -l app=" + IMAGE_NAME + " -o=name --no-headers | tr -d '\n'").getStdout();
Container.ExecResult execResult = K3S.execInContainer("sh", "-c", "kubectl logs " + appPodName.trim());
return execResult.getStdout();
}
catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
private WebClient.Builder builder() {
return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create()));
}
private RetryBackoffSpec retrySpec() {
return Retry.fixedDelay(120, Duration.ofSeconds(2)).filter(Objects::nonNull);
}
}

View File

@@ -6,4 +6,4 @@ metadata:
data:
# from.properties.key=initial
application.properties: |
ZnJvbS5wcm9wZXJ0aWVzLmtleT1pbml0aWFsCg==
ZnJvbS5wcm9wZXJ0aWVzLmtleT1pbml0aWFs