Fabric8 configmap event reload patch refactor part 3 (#1467)

This commit is contained in:
erabii
2023-10-06 22:38:52 +03:00
committed by GitHub
parent 282649343b
commit 4ab2406da5
46 changed files with 524 additions and 954 deletions

View File

@@ -52,9 +52,8 @@
<module>spring-cloud-kubernetes-fabric8-istio-it</module>
<module>spring-cloud-kubernetes-fabric8-client-discovery</module>
<module>spring-cloud-kubernetes-fabric8-client-loadbalancer</module>
<module>spring-cloud-kubernetes-fabric8-client-configmap-polling-reload</module>
<module>spring-cloud-kubernetes-fabric8-client-secrets-event-reload</module>
<module>spring-cloud-kubernetes-fabric8-client-configmap-event-reload</module>
<module>spring-cloud-kubernetes-fabric8-client-reload</module>
<module>spring-cloud-kubernetes-discoveryclient-it</module>
<module>spring-cloud-kubernetes-client-loadbalancer-it</module>

View File

@@ -1,113 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>spring-cloud-kubernetes-integration-tests</artifactId>
<groupId>org.springframework.cloud</groupId>
<version>3.0.5-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>spring-cloud-kubernetes-fabric8-client-configmap-event-reload</artifactId>
<dependencies>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-kubernetes-fabric8-config</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-webflux</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-bootstrap</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-kubernetes-test-support</artifactId>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>../src/main/resources</directory>
<filtering>true</filtering>
</resource>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<!-- build image in the 'package' phase, and ignore plain tests -->
<!-- via maven-surefire-plugin::skipTests -->
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<imageName>docker.io/springcloud/${project.artifactId}:${project.version}</imageName>
</configuration>
<executions>
<execution>
<id>build-image</id>
<configuration>
<skip>${skip.build.image}</skip>
</configuration>
<phase>package</phase>
<goals>
<goal>build-image</goal>
</goals>
</execution>
<execution>
<id>repackage</id>
<phase>package</phase>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- ignore plain tests (in the 'test' phase), so that we could build the image first, see above -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
<!-- run tests in the 'integration-tests' phase, one that is after 'package' (where we build the image) -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>integration-test</goal>
</goals>
</execution>
</executions>
<configuration>
<includes>
<include>${testsToRun}</include>
</includes>
</configuration>
</plugin>
</plugins>
</build>
</project>

View File

@@ -1,14 +0,0 @@
apiVersion: v1
kind: Service
metadata:
labels:
app: spring-cloud-kubernetes-fabric8-client-configmap-event-reload
name: spring-cloud-kubernetes-fabric8-client-configmap-event-reload
spec:
ports:
- name: http
port: 8080
targetPort: 8080
selector:
app: spring-cloud-kubernetes-fabric8-client-configmap-event-reload
type: ClusterIP

View File

@@ -1,35 +0,0 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.configmap.polling.reload;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
/**
* @author wind57
*/
@SpringBootApplication
@EnableConfigurationProperties(ConfigMapProperties.class)
public class ConfigMapApp {
public static void main(String[] args) {
SpringApplication.run(ConfigMapApp.class, args);
}
}

View File

@@ -1,39 +0,0 @@
/*
* Copyright 2013-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.configmap.polling.reload;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* @author wind57
*/
@RestController
public class ConfigMapController {
private final ConfigMapProperties properties;
public ConfigMapController(ConfigMapProperties properties) {
this.properties = properties;
}
@GetMapping("/key")
public String key() {
return properties.getKey();
}
}

View File

@@ -1,195 +0,0 @@
/*
* Copyright 2013-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.configmap.polling.reload;
import java.io.InputStream;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.EnvVarBuilder;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.networking.v1.Ingress;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.utils.Serialization;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.Container;
import org.testcontainers.k3s.K3sContainer;
import reactor.netty.http.client.HttpClient;
import reactor.util.retry.Retry;
import reactor.util.retry.RetryBackoffSpec;
import org.springframework.cloud.kubernetes.integration.tests.commons.Commons;
import org.springframework.cloud.kubernetes.integration.tests.commons.Phase;
import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util;
import org.springframework.http.HttpMethod;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.web.reactive.function.client.WebClient;
import static org.awaitility.Awaitility.await;
public class BootstrapEnabledPollingReloadConfigMapMountIT {
private static final String IMAGE_NAME = "spring-cloud-kubernetes-fabric8-client-configmap-polling-reload";
private static final String NAMESPACE = "default";
private static Util util;
private static KubernetesClient client;
private static final K3sContainer K3S = Commons.container();
@BeforeAll
static void beforeAll() throws Exception {
K3S.start();
Commons.validateImage(IMAGE_NAME, K3S);
Commons.loadSpringCloudKubernetesImage(IMAGE_NAME, K3S);
util = new Util(K3S);
client = util.client();
util.setUp(NAMESPACE);
manifests(Phase.CREATE);
}
@AfterAll
static void after() throws Exception {
manifests(Phase.DELETE);
Commons.cleanUp(IMAGE_NAME, K3S);
Commons.systemPrune();
}
/**
* <pre>
* - we have bootstrap enabled, which means we will 'locate' property sources
* from config maps.
* - there are no explicit config maps to search for, but what we will also read,
* is 'spring.cloud.kubernetes.config.paths', which we have set to
* '/tmp/application.properties'
* in this test. That is populated by the volumeMounts (see deployment-mount.yaml)
* - we first assert that we are actually reading the path based source via (1), (2) and (3).
*
* - we then change the config map content, wait for k8s to pick it up and replace them
* - our polling will then detect that change, and trigger a reload.
* </pre>
*/
@Test
void test() {
String logs = logs();
// (1)
Assertions.assertTrue(logs.contains("paths property sources : [/tmp/application.properties]"));
// (2)
Assertions.assertTrue(logs.contains("will add file-based property source : /tmp/application.properties"));
// (3)
WebClient webClient = builder().baseUrl("http://localhost/key").build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec())
.block();
// we first read the initial value from the configmap
Assertions.assertEquals("as-mount-initial", result);
// replace data in configmap and wait for k8s to pick it up
// our polling will detect that and restart the app
InputStream configMapStream = util.inputStream("mount/configmap-mount.yaml");
ConfigMap configMap = Serialization.unmarshal(configMapStream, ConfigMap.class);
configMap.setData(Map.of("application.properties", "from.properties.key=as-mount-changed"));
client.configMaps().inNamespace("default").resource(configMap).createOrReplace();
await().timeout(Duration.ofSeconds(360)).until(() -> webClient.method(HttpMethod.GET).retrieve()
.bodyToMono(String.class).retryWhen(retrySpec()).block().equals("as-mount-changed"));
}
private static void manifests(Phase phase) {
InputStream deploymentStream = util.inputStream("mount/deployment-mount.yaml");
InputStream serviceStream = util.inputStream("service.yaml");
InputStream ingressStream = util.inputStream("ingress.yaml");
InputStream configMapStream = util.inputStream("mount/configmap-mount.yaml");
Deployment deployment = Serialization.unmarshal(deploymentStream, Deployment.class);
Service service = Serialization.unmarshal(serviceStream, Service.class);
Ingress ingress = Serialization.unmarshal(ingressStream, Ingress.class);
ConfigMap configMap = Serialization.unmarshal(configMapStream, ConfigMap.class);
List<EnvVar> existing = new ArrayList<>(
deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv());
// bootstrap is enabled, which means that in 'application-with-bootstrap.yaml',
// config-data support is disabled.
EnvVar withBootstrapActiveProfile = new EnvVarBuilder().withName("SPRING_PROFILES_ACTIVE")
.withValue("with-bootstrap").build();
EnvVar enabledBootstrap = new EnvVarBuilder().withName("SPRING_CLOUD_BOOTSTRAP_ENABLED").withValue("TRUE")
.build();
EnvVar debugLevelReloadCommons = new EnvVarBuilder()
.withName("LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_COMMONS_CONFIG_RELOAD").withValue("DEBUG")
.build();
EnvVar debugLevelConfig = new EnvVarBuilder()
.withName("LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_COMMONS_CONFIG").withValue("DEBUG")
.build();
EnvVar debugLevelCommons = new EnvVarBuilder()
.withName("LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_COMMONS").withValue("DEBUG").build();
existing.add(withBootstrapActiveProfile);
existing.add(enabledBootstrap);
existing.add(debugLevelReloadCommons);
existing.add(debugLevelCommons);
existing.add(debugLevelConfig);
deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(existing);
if (phase.equals(Phase.CREATE)) {
util.createAndWait(NAMESPACE, configMap, null);
util.createAndWait(NAMESPACE, null, deployment, service, ingress, true);
}
else {
util.deleteAndWait(NAMESPACE, configMap, null);
util.deleteAndWait(NAMESPACE, deployment, service, ingress);
}
}
private WebClient.Builder builder() {
return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create()));
}
private RetryBackoffSpec retrySpec() {
return Retry.fixedDelay(60, Duration.ofSeconds(1)).filter(Objects::nonNull);
}
private String logs() {
try {
String appPodName = K3S.execInContainer("sh", "-c",
"kubectl get pods -l app=" + IMAGE_NAME + " -o=name --no-headers | tr -d '\n'").getStdout();
Container.ExecResult execResult = K3S.execInContainer("sh", "-c", "kubectl logs " + appPodName.trim());
return execResult.getStdout();
}
catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
}

View File

@@ -1,145 +0,0 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.configmap.polling.reload;
import java.io.InputStream;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.api.model.ConfigMapBuilder;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.EnvVarBuilder;
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.networking.v1.Ingress;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.utils.Serialization;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.testcontainers.k3s.K3sContainer;
import reactor.netty.http.client.HttpClient;
import reactor.util.retry.Retry;
import reactor.util.retry.RetryBackoffSpec;
import org.springframework.cloud.kubernetes.integration.tests.commons.Commons;
import org.springframework.cloud.kubernetes.integration.tests.commons.Phase;
import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util;
import org.springframework.http.HttpMethod;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.web.reactive.function.client.WebClient;
import static org.awaitility.Awaitility.await;
/**
* @author wind57
*/
class ConfigMapPollingReloadIT {
private static final String IMAGE_NAME = "spring-cloud-kubernetes-fabric8-client-configmap-polling-reload";
private static final String NAMESPACE = "default";
private static Util util;
private static KubernetesClient client;
private static final K3sContainer K3S = Commons.container();
@BeforeAll
static void beforeAll() throws Exception {
K3S.start();
Commons.validateImage(IMAGE_NAME, K3S);
Commons.loadSpringCloudKubernetesImage(IMAGE_NAME, K3S);
util = new Util(K3S);
client = util.client();
util.setUp(NAMESPACE);
manifests(Phase.CREATE);
}
@AfterAll
static void after() throws Exception {
manifests(Phase.DELETE);
Commons.cleanUp(IMAGE_NAME, K3S);
Commons.systemPrune();
}
@Test
void test() {
WebClient webClient = builder().baseUrl("http://localhost/key").build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec())
.block();
// we first read the initial value from the configmap
Assertions.assertEquals("initial", result);
// then deploy a new version of configmap
// since we poll and have reload in place, the new property must be visible
ConfigMap map = new ConfigMapBuilder()
.withMetadata(new ObjectMetaBuilder().withNamespace("default").withName("poll-reload").build())
.withData(Map.of("application.properties", "from.properties.key=after-change")).build();
client.configMaps().inNamespace("default").resource(map).createOrReplace();
await().timeout(Duration.ofSeconds(60)).until(() -> webClient.method(HttpMethod.GET).retrieve()
.bodyToMono(String.class).retryWhen(retrySpec()).block().equals("after-change"));
}
private static void manifests(Phase phase) {
InputStream deploymentStream = util.inputStream("deployment.yaml");
InputStream serviceStream = util.inputStream("service.yaml");
InputStream ingressStream = util.inputStream("ingress.yaml");
InputStream configMapStream = util.inputStream("configmap.yaml");
Deployment deployment = Serialization.unmarshal(deploymentStream, Deployment.class);
Service service = Serialization.unmarshal(serviceStream, Service.class);
Ingress ingress = Serialization.unmarshal(ingressStream, Ingress.class);
ConfigMap configMap = Serialization.unmarshal(configMapStream, ConfigMap.class);
List<EnvVar> existing = new ArrayList<>(
deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv());
existing.add(new EnvVarBuilder().withName("SPRING_PROFILES_ACTIVE").withValue("no-mount").build());
deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(existing);
if (phase.equals(Phase.CREATE)) {
util.createAndWait(NAMESPACE, configMap, null);
util.createAndWait(NAMESPACE, null, deployment, service, ingress, true);
}
else {
util.deleteAndWait(NAMESPACE, configMap, null);
util.deleteAndWait(NAMESPACE, deployment, service, ingress);
}
}
private WebClient.Builder builder() {
return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create()));
}
private RetryBackoffSpec retrySpec() {
return Retry.fixedDelay(60, Duration.ofSeconds(1)).filter(Objects::nonNull);
}
}

View File

@@ -1,198 +0,0 @@
/*
* Copyright 2013-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.configmap.polling.reload;
import java.io.InputStream;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.EnvVarBuilder;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.networking.v1.Ingress;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.utils.Serialization;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.Container;
import org.testcontainers.k3s.K3sContainer;
import reactor.netty.http.client.HttpClient;
import reactor.util.retry.Retry;
import reactor.util.retry.RetryBackoffSpec;
import org.springframework.cloud.kubernetes.integration.tests.commons.Commons;
import org.springframework.cloud.kubernetes.integration.tests.commons.Phase;
import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util;
import org.springframework.http.HttpMethod;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.web.reactive.function.client.WebClient;
import static org.awaitility.Awaitility.await;
/**
* @author wind57
*/
class PollingReloadConfigMapMountIT {
private static final String IMAGE_NAME = "spring-cloud-kubernetes-fabric8-client-configmap-polling-reload";
private static final String NAMESPACE = "default";
private static Util util;
private static KubernetesClient client;
private static final K3sContainer K3S = Commons.container();
@BeforeAll
static void beforeAll() throws Exception {
K3S.start();
Commons.validateImage(IMAGE_NAME, K3S);
Commons.loadSpringCloudKubernetesImage(IMAGE_NAME, K3S);
util = new Util(K3S);
client = util.client();
util.setUp(NAMESPACE);
manifests(Phase.CREATE);
}
@AfterAll
static void after() throws Exception {
manifests(Phase.DELETE);
Commons.cleanUp(IMAGE_NAME, K3S);
Commons.systemPrune();
}
/**
* <pre>
* - we have "spring.config.import: kubernetes", which means we will 'locate' property sources
* from config maps.
* - the property above means that at the moment we will be searching for config maps that only
* match the application name, in this specific test there is no such config map.
* - what we will also read, is 'spring.cloud.kubernetes.config.paths', which we have set to
* '/tmp/application.properties'
* in this test. That is populated by the volumeMounts (see deployment-mount.yaml)
* - we first assert that we are actually reading the path based source via (1), (2) and (3).
*
* - we then change the config map content, wait for k8s to pick it up and replace them
* - our polling will then detect that change, and trigger a reload.
* </pre>
*/
@Test
void test() {
String logs = logs();
// (1)
Assertions.assertTrue(logs.contains("paths property sources : [/tmp/application.properties]"));
// (2)
Assertions.assertTrue(logs.contains("will add file-based property source : /tmp/application.properties"));
// (3)
WebClient webClient = builder().baseUrl("http://localhost/key").build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec())
.block();
// we first read the initial value from the configmap
Assertions.assertEquals("as-mount-initial", result);
// replace data in configmap and wait for k8s to pick it up
// our polling will detect that and restart the app
InputStream configMapStream = util.inputStream("mount/configmap-mount.yaml");
ConfigMap configMap = Serialization.unmarshal(configMapStream, ConfigMap.class);
configMap.setData(Map.of("application.properties", "from.properties.key=as-mount-changed"));
client.configMaps().inNamespace("default").resource(configMap).createOrReplace();
await().timeout(Duration.ofSeconds(360)).until(() -> webClient.method(HttpMethod.GET).retrieve()
.bodyToMono(String.class).retryWhen(retrySpec()).block().equals("as-mount-changed"));
}
private static void manifests(Phase phase) {
InputStream deploymentStream = util.inputStream("mount/deployment-mount.yaml");
InputStream serviceStream = util.inputStream("service.yaml");
InputStream ingressStream = util.inputStream("ingress.yaml");
InputStream configMapStream = util.inputStream("mount/configmap-mount.yaml");
Deployment deployment = Serialization.unmarshal(deploymentStream, Deployment.class);
Service service = Serialization.unmarshal(serviceStream, Service.class);
Ingress ingress = Serialization.unmarshal(ingressStream, Ingress.class);
ConfigMap configMap = Serialization.unmarshal(configMapStream, ConfigMap.class);
List<EnvVar> existing = new ArrayList<>(
deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv());
// bootstrap is disabled, which means that in 'application-mount.yaml',
// config-data support is enabled.
EnvVar mountActiveProfile = new EnvVarBuilder().withName("SPRING_PROFILES_ACTIVE").withValue("mount").build();
EnvVar disableBootstrap = new EnvVarBuilder().withName("SPRING_CLOUD_BOOTSTRAP_ENABLED").withValue("FALSE")
.build();
EnvVar debugLevelReloadCommons = new EnvVarBuilder()
.withName("LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_COMMONS_CONFIG_RELOAD").withValue("DEBUG")
.build();
EnvVar debugLevelConfig = new EnvVarBuilder()
.withName("LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_COMMONS_CONFIG").withValue("DEBUG")
.build();
EnvVar debugLevelCommons = new EnvVarBuilder()
.withName("LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_COMMONS").withValue("DEBUG").build();
existing.add(mountActiveProfile);
existing.add(disableBootstrap);
existing.add(debugLevelReloadCommons);
existing.add(debugLevelCommons);
existing.add(debugLevelConfig);
deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(existing);
if (phase.equals(Phase.CREATE)) {
util.createAndWait(NAMESPACE, configMap, null);
util.createAndWait(NAMESPACE, null, deployment, service, ingress, true);
}
else {
util.deleteAndWait(NAMESPACE, configMap, null);
util.deleteAndWait(NAMESPACE, deployment, service, ingress);
}
}
private WebClient.Builder builder() {
return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create()));
}
private RetryBackoffSpec retrySpec() {
return Retry.fixedDelay(60, Duration.ofSeconds(1)).filter(Objects::nonNull);
}
private String logs() {
try {
String appPodName = K3S.execInContainer("sh", "-c",
"kubectl get pods -l app=" + IMAGE_NAME + " -o=name --no-headers | tr -d '\n'").getStdout();
Container.ExecResult execResult = K3S.execInContainer("sh", "-c", "kubectl logs " + appPodName.trim());
return execResult.getStdout();
}
catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
}

View File

@@ -1,28 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: spring-cloud-kubernetes-fabric8-client-configmap-deployment-polling-reload
spec:
selector:
matchLabels:
app: spring-cloud-kubernetes-fabric8-client-configmap-polling-reload
template:
metadata:
labels:
app: spring-cloud-kubernetes-fabric8-client-configmap-polling-reload
spec:
serviceAccountName: spring-cloud-kubernetes-serviceaccount
containers:
- name: spring-cloud-kubernetes-fabric8-client-configmap-polling-reload
image: docker.io/springcloud/spring-cloud-kubernetes-fabric8-client-configmap-polling-reload
imagePullPolicy: IfNotPresent
readinessProbe:
httpGet:
port: 8080
path: /actuator/health/readiness
livenessProbe:
httpGet:
port: 8080
path: /actuator/health/liveness
ports:
- containerPort: 8080

View File

@@ -1,16 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: spring-cloud-kubernetes-fabric8-client-configmap-ingress-polling-reload
namespace: default
spec:
rules:
- http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: spring-cloud-kubernetes-fabric8-client-configmap-polling-reload
port:
number: 8080

View File

@@ -1,15 +0,0 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="STDOUT"/>
</root>
<logger name="org.testcontainers" level="INFO"/>
<logger name="com.github.dockerjava" level="WARN"/>
<logger name="io.fabric8.kubernetes.client" level="ERROR"/>
</configuration>

View File

@@ -1,8 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: poll-reload-as-mount
namespace: default
data:
application.properties: |
from.properties.key=as-mount-initial

View File

@@ -1,37 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: spring-cloud-kubernetes-fabric8-client-configmap-deployment-polling-reload
spec:
selector:
matchLabels:
app: spring-cloud-kubernetes-fabric8-client-configmap-polling-reload
template:
metadata:
labels:
app: spring-cloud-kubernetes-fabric8-client-configmap-polling-reload
spec:
serviceAccountName: spring-cloud-kubernetes-serviceaccount
containers:
- name: spring-cloud-kubernetes-fabric8-client-configmap-polling-reload
image: docker.io/springcloud/spring-cloud-kubernetes-fabric8-client-configmap-polling-reload
imagePullPolicy: IfNotPresent
readinessProbe:
httpGet:
port: 8080
path: /actuator/health/readiness
livenessProbe:
httpGet:
port: 8080
path: /actuator/health/liveness
ports:
- containerPort: 8080
volumeMounts:
- name: config-map-volume
mountPath: /tmp
volumes:
- name: config-map-volume
configMap:
name: poll-reload-as-mount

View File

@@ -1,14 +0,0 @@
apiVersion: v1
kind: Service
metadata:
labels:
app: spring-cloud-kubernetes-fabric8-client-configmap-polling-reload
name: spring-cloud-kubernetes-fabric8-client-configmap-polling-reload
spec:
ports:
- name: http
port: 8080
targetPort: 8080
selector:
app: spring-cloud-kubernetes-fabric8-client-configmap-polling-reload
type: ClusterIP

View File

@@ -3,13 +3,13 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-kubernetes-integration-tests</artifactId>
<groupId>org.springframework.cloud</groupId>
<version>3.0.5-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>spring-cloud-kubernetes-fabric8-client-configmap-polling-reload</artifactId>
<artifactId>spring-cloud-kubernetes-fabric8-client-reload</artifactId>
<dependencies>

View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.configmap.event.reload;
package org.springframework.cloud.kubernetes.fabric8.reload;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -24,7 +24,8 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
* @author wind57
*/
@SpringBootApplication
@EnableConfigurationProperties({ LeftProperties.class, RightProperties.class, RightWithLabelsProperties.class })
@EnableConfigurationProperties({ LeftProperties.class, RightProperties.class, RightWithLabelsProperties.class,
ConfigMapProperties.class })
public class App {
public static void main(String[] args) {

View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.configmap.polling.reload;
package org.springframework.cloud.kubernetes.fabric8.reload;
import org.springframework.boot.context.properties.ConfigurationProperties;

View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.configmap.event.reload;
package org.springframework.cloud.kubernetes.fabric8.reload;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
@@ -31,11 +31,14 @@ public class Controller {
private final RightWithLabelsProperties rightWithLabelsProperties;
private final ConfigMapProperties configMapProperties;
public Controller(LeftProperties leftProperties, RightProperties rightProperties,
RightWithLabelsProperties rightWithLabelsProperties) {
RightWithLabelsProperties rightWithLabelsProperties, ConfigMapProperties configMapProperties) {
this.leftProperties = leftProperties;
this.rightProperties = rightProperties;
this.rightWithLabelsProperties = rightWithLabelsProperties;
this.configMapProperties = configMapProperties;
}
@GetMapping("/left")
@@ -53,4 +56,9 @@ public class Controller {
return rightWithLabelsProperties.getValue();
}
@GetMapping("/key")
public String key() {
return configMapProperties.getKey();
}
}

View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.configmap.event.reload;
package org.springframework.cloud.kubernetes.fabric8.reload;
import org.springframework.boot.context.properties.ConfigurationProperties;

View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.configmap.event.reload;
package org.springframework.cloud.kubernetes.fabric8.reload;
import org.springframework.boot.context.properties.ConfigurationProperties;

View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.configmap.event.reload;
package org.springframework.cloud.kubernetes.fabric8.reload;
import org.springframework.boot.context.properties.ConfigurationProperties;

View File

@@ -0,0 +1,79 @@
/*
* Copyright 2013-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.reload;
import java.io.InputStream;
import java.time.Duration;
import java.util.Map;
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.utils.Serialization;
import org.junit.jupiter.api.Assertions;
import org.testcontainers.k3s.K3sContainer;
import org.springframework.cloud.kubernetes.integration.tests.commons.Commons;
import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util;
import org.springframework.http.HttpMethod;
import org.springframework.web.reactive.function.client.WebClient;
import static org.awaitility.Awaitility.await;
final class BootstrapEnabledPollingReloadConfigMapMountDelegate {
/**
* <pre>
* - we have bootstrap enabled, which means we will 'locate' property sources
* from config maps.
* - there are no explicit config maps to search for, but what we will also read,
* is 'spring.cloud.kubernetes.config.paths', which we have set to
* '/tmp/application.properties'
* in this test. That is populated by the volumeMounts (see deployment-mount.yaml)
* - we first assert that we are actually reading the path based source via (1), (2) and (3).
*
* - we then change the config map content, wait for k8s to pick it up and replace them
* - our polling will then detect that change, and trigger a reload.
* </pre>
*/
static void testPollingReloadConfigMapWithBootstrap(KubernetesClient client, Util util, K3sContainer container,
String appLabelValue) {
// (1)
Commons.waitForLogStatement("paths property sources : [/tmp/application.properties]", container, appLabelValue);
// (2)
Commons.waitForLogStatement("will add file-based property source : /tmp/application.properties", container,
appLabelValue);
// (3)
WebClient webClient = TestUtil.builder().baseUrl("http://localhost/key").build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(TestUtil.retrySpec())
.block();
// we first read the initial value from the configmap
Assertions.assertEquals("as-mount-initial", result);
// replace data in configmap and wait for k8s to pick it up
// our polling will detect that and restart the app
InputStream configMapStream = util.inputStream("configmap.yaml");
ConfigMap configMap = Serialization.unmarshal(configMapStream, ConfigMap.class);
configMap.setData(Map.of("application.properties", "from.properties.key=as-mount-changed"));
client.configMaps().inNamespace("default").resource(configMap).createOrReplace();
await().timeout(Duration.ofSeconds(360)).until(() -> webClient.method(HttpMethod.GET).retrieve()
.bodyToMono(String.class).retryWhen(TestUtil.retrySpec()).block().equals("as-mount-changed"));
}
}

View File

@@ -0,0 +1,83 @@
/*
* Copyright 2013-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.reload;
import java.io.InputStream;
import java.time.Duration;
import java.util.Map;
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.utils.Serialization;
import org.junit.jupiter.api.Assertions;
import org.testcontainers.k3s.K3sContainer;
import org.springframework.cloud.kubernetes.integration.tests.commons.Commons;
import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util;
import org.springframework.http.HttpMethod;
import org.springframework.web.reactive.function.client.WebClient;
import static org.awaitility.Awaitility.await;
/**
* @author wind57
*/
final class ConfigMapMountPollingReloadDelegate {
/**
* <pre>
* - we have "spring.config.import: kubernetes", which means we will 'locate' property sources
* from config maps.
* - the property above means that at the moment we will be searching for config maps that only
* match the application name, in this specific test there is no such config map.
* - what we will also read, is 'spring.cloud.kubernetes.config.paths', which we have set to
* '/tmp/application.properties'
* in this test. That is populated by the volumeMounts (see deployment-mount.yaml)
* - we first assert that we are actually reading the path based source via (1), (2) and (3).
*
* - we then change the config map content, wait for k8s to pick it up and replace them
* - our polling will then detect that change, and trigger a reload.
* </pre>
*/
static void testConfigMapMountPollingReload(KubernetesClient client, Util util, K3sContainer container,
String appLabelValue) {
// (1)
Commons.waitForLogStatement("paths property sources : [/tmp/application.properties]", container, appLabelValue);
// (2)
Commons.waitForLogStatement("will add file-based property source : /tmp/application.properties", container,
appLabelValue);
// (3)
WebClient webClient = TestUtil.builder().baseUrl("http://localhost/key").build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(TestUtil.retrySpec())
.block();
// we first read the initial value from the configmap
Assertions.assertEquals("as-mount-initial", result);
// replace data in configmap and wait for k8s to pick it up
// our polling will detect that and restart the app
InputStream configMapStream = util.inputStream("configmap.yaml");
ConfigMap configMap = Serialization.unmarshal(configMapStream, ConfigMap.class);
configMap.setData(Map.of("application.properties", "from.properties.key=as-mount-changed"));
client.configMaps().inNamespace("default").resource(configMap).createOrReplace();
await().timeout(Duration.ofSeconds(360)).until(() -> webClient.method(HttpMethod.GET).retrieve()
.bodyToMono(String.class).retryWhen(TestUtil.retrySpec()).block().equals("as-mount-changed"));
}
}

View File

@@ -0,0 +1,61 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.reload;
import java.time.Duration;
import java.util.Map;
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.api.model.ConfigMapBuilder;
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
import io.fabric8.kubernetes.client.KubernetesClient;
import org.junit.jupiter.api.Assertions;
import org.springframework.http.HttpMethod;
import org.springframework.web.client.HttpServerErrorException;
import org.springframework.web.reactive.function.client.WebClient;
import static org.awaitility.Awaitility.await;
/**
* @author wind57
*/
final class ConfigMapPollingReloadDelegate {
static void testConfigMapPollingReload(KubernetesClient client) {
WebClient webClient = TestUtil.builder().baseUrl("http://localhost/key").build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(TestUtil.retrySpec())
.block();
// we first read the initial value from the configmap
Assertions.assertEquals("as-mount-initial", result);
// then deploy a new version of configmap
// since we poll and have reload in place, the new property must be visible
ConfigMap map = new ConfigMapBuilder()
.withMetadata(new ObjectMetaBuilder().withNamespace("default").withName("poll-reload").build())
.withData(Map.of("application.properties", "from.properties.key=after-change")).build();
client.configMaps().inNamespace("default").resource(map).createOrReplace();
await().ignoreException(HttpServerErrorException.BadGateway.class).timeout(Duration.ofSeconds(120))
.until(() -> webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(TestUtil.retrySpec())
.block().equals("after-change"));
}
}

View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.configmap.event.reload;
package org.springframework.cloud.kubernetes.fabric8.reload;
import java.time.Duration;
import java.util.Map;
@@ -24,7 +24,6 @@ import io.fabric8.kubernetes.api.model.ConfigMapBuilder;
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
import io.fabric8.kubernetes.client.KubernetesClient;
import org.junit.jupiter.api.Assertions;
import org.testcontainers.containers.Container;
import org.testcontainers.k3s.K3sContainer;
import org.springframework.cloud.kubernetes.integration.tests.commons.Commons;
@@ -32,18 +31,17 @@ import org.springframework.http.HttpMethod;
import org.springframework.web.reactive.function.client.WebClient;
import static org.awaitility.Awaitility.await;
import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.builder;
import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.replaceConfigMap;
import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.retrySpec;
import static org.springframework.cloud.kubernetes.fabric8.reload.TestUtil.builder;
import static org.springframework.cloud.kubernetes.fabric8.reload.TestUtil.logs;
import static org.springframework.cloud.kubernetes.fabric8.reload.TestUtil.replaceConfigMap;
import static org.springframework.cloud.kubernetes.fabric8.reload.TestUtil.retrySpec;
final class DataChangesInConfigMapReloadDelegate {
private static final String IMAGE_NAME = "spring-cloud-kubernetes-fabric8-client-configmap-event-reload";
private static final String IMAGE_NAME = "spring-cloud-kubernetes-fabric8-client-reload";
private static final String LEFT_NAMESPACE = "left";
private static final K3sContainer K3S = Commons.container();
/**
* <pre>
* - configMap with no labels and data: left.value = left-initial exists in namespace left
@@ -55,7 +53,7 @@ final class DataChangesInConfigMapReloadDelegate {
* - then we change data inside the config map, and we must see the updated value
* </pre>
*/
static void testDataChangesInConfigMap(KubernetesClient client) {
static void testDataChangesInConfigMap(KubernetesClient client, K3sContainer container, String appLabelValue) {
Commons.assertReloadLogStatements("added configmap informer for namespace",
"added secret informer for namespace", IMAGE_NAME);
@@ -81,7 +79,7 @@ final class DataChangesInConfigMapReloadDelegate {
return "left-initial".equals(innerResult);
});
String logs = logs();
String logs = logs(container, appLabelValue);
Assertions.assertTrue(logs.contains("ConfigMap left-configmap was updated in namespace left"));
Assertions.assertTrue(logs.contains("data in configmap has not changed, will not reload"));
@@ -102,18 +100,4 @@ final class DataChangesInConfigMapReloadDelegate {
}
private static String logs() {
try {
String appPodName = K3S.execInContainer("sh", "-c",
"kubectl get pods -l app=" + IMAGE_NAME + " -o=name --no-headers | tr -d '\n'").getStdout();
Container.ExecResult execResult = K3S.execInContainer("sh", "-c", "kubectl logs " + appPodName.trim());
return execResult.getStdout();
}
catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
}

View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.configmap.event.reload;
package org.springframework.cloud.kubernetes.fabric8.reload;
import java.io.InputStream;
import java.time.Duration;
@@ -42,22 +42,14 @@ import org.springframework.http.HttpMethod;
import org.springframework.web.reactive.function.client.WebClient;
import static org.awaitility.Awaitility.await;
import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.builder;
import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.patchFour;
import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.patchOne;
import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.patchThree;
import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.patchTwo;
import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.reCreateConfigMaps;
import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.replaceConfigMap;
import static org.springframework.cloud.kubernetes.fabric8.configmap.event.reload.TestUtil.retrySpec;
import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.pomVersion;
/**
* @author wind57
*/
class ConfigMapEventReloadIT {
class Fabric8EventReloadIT {
private static final String IMAGE_NAME = "spring-cloud-kubernetes-fabric8-client-configmap-event-reload";
private static final String IMAGE_NAME = "spring-cloud-kubernetes-fabric8-client-reload";
private static final String DOCKER_IMAGE = "docker.io/springcloud/" + IMAGE_NAME + ":" + pomVersion();
@@ -81,6 +73,7 @@ class ConfigMapEventReloadIT {
util.createNamespace("left");
util.createNamespace("right");
util.setUpClusterWide(NAMESPACE, Set.of("left", "right"));
util.setUp(NAMESPACE);
manifests(Phase.CREATE);
}
@@ -108,16 +101,16 @@ class ConfigMapEventReloadIT {
Commons.assertReloadLogStatements("added configmap informer for namespace",
"added secret informer for namespace", IMAGE_NAME);
WebClient webClient = builder().baseUrl("http://localhost/left").build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec())
WebClient webClient = TestUtil.builder().baseUrl("http://localhost/left").build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(TestUtil.retrySpec())
.block();
// we first read the initial value from the left-configmap
Assertions.assertEquals("left-initial", result);
// then read the value from the right-configmap
webClient = builder().baseUrl("http://localhost/right").build();
result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec()).block();
webClient = TestUtil.builder().baseUrl("http://localhost/right").build();
result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(TestUtil.retrySpec()).block();
Assertions.assertEquals("right-initial", result);
// then deploy a new version of right-configmap
@@ -125,14 +118,14 @@ class ConfigMapEventReloadIT {
.withMetadata(new ObjectMetaBuilder().withNamespace("right").withName("right-configmap").build())
.withData(Map.of("right.value", "right-after-change")).build();
replaceConfigMap(client, rightConfigMapAfterChange, "right");
TestUtil.replaceConfigMap(client, rightConfigMapAfterChange, "right");
webClient = builder().baseUrl("http://localhost/left").build();
webClient = TestUtil.builder().baseUrl("http://localhost/left").build();
WebClient finalWebClient = webClient;
await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(30)).until(() -> {
String innerResult = finalWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
.retryWhen(retrySpec()).block();
.retryWhen(TestUtil.retrySpec()).block();
// left configmap has not changed, no restart of app has happened
return "left-initial".equals(innerResult);
});
@@ -141,6 +134,9 @@ class ConfigMapEventReloadIT {
testInform();
testInformFromOneNamespaceEventTriggeredSecretsDisabled();
testDataChangesInConfigMap();
testConfigMapPollingReload();
testConfigMapMountPollingReload();
testPollingReloadConfigMapWithBootstrap();
}
/**
@@ -154,15 +150,15 @@ class ConfigMapEventReloadIT {
*/
void testInformFromOneNamespaceEventTriggered() {
reCreateConfigMaps(util, client);
patchOne(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE);
TestUtil.reCreateConfigMaps(util, client);
TestUtil.patchOne(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE);
Commons.assertReloadLogStatements("added configmap informer for namespace",
"added secret informer for namespace", IMAGE_NAME);
// read the value from the right-configmap
WebClient webClient = builder().baseUrl("http://localhost/right").build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec())
WebClient webClient = TestUtil.builder().baseUrl("http://localhost/right").build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(TestUtil.retrySpec())
.block();
Assertions.assertEquals("right-initial", result);
@@ -171,13 +167,13 @@ class ConfigMapEventReloadIT {
.withMetadata(new ObjectMetaBuilder().withNamespace("right").withName("right-configmap").build())
.withData(Map.of("right.value", "right-after-change")).build();
replaceConfigMap(client, rightConfigMapAfterChange, "right");
TestUtil.replaceConfigMap(client, rightConfigMapAfterChange, "right");
String[] resultAfterChange = new String[1];
await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> {
WebClient innerWebClient = builder().baseUrl("http://localhost/right").build();
WebClient innerWebClient = TestUtil.builder().baseUrl("http://localhost/right").build();
String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
.retryWhen(retrySpec()).block();
.retryWhen(TestUtil.retrySpec()).block();
resultAfterChange[0] = innerResult;
return innerResult != null;
});
@@ -196,22 +192,22 @@ class ConfigMapEventReloadIT {
*/
void testInform() {
reCreateConfigMaps(util, client);
patchTwo(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE);
TestUtil.reCreateConfigMaps(util, client);
TestUtil.patchTwo(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE);
Commons.assertReloadLogStatements("added configmap informer for namespace",
"added secret informer for namespace", IMAGE_NAME);
// read the initial value from the right-configmap
WebClient rightWebClient = builder().baseUrl("http://localhost/right").build();
WebClient rightWebClient = TestUtil.builder().baseUrl("http://localhost/right").build();
String rightResult = rightWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
.retryWhen(retrySpec()).block();
.retryWhen(TestUtil.retrySpec()).block();
Assertions.assertEquals("right-initial", rightResult);
// then read the initial value from the right-with-label-configmap
WebClient rightWithLabelWebClient = builder().baseUrl("http://localhost/with-label").build();
WebClient rightWithLabelWebClient = TestUtil.builder().baseUrl("http://localhost/with-label").build();
String rightWithLabelResult = rightWithLabelWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
.retryWhen(retrySpec()).block();
.retryWhen(TestUtil.retrySpec()).block();
Assertions.assertEquals("right-with-label-initial", rightWithLabelResult);
// then deploy a new version of right-configmap
@@ -219,12 +215,12 @@ class ConfigMapEventReloadIT {
.withMetadata(new ObjectMetaBuilder().withNamespace("right").withName("right-configmap").build())
.withData(Map.of("right.value", "right-after-change")).build();
replaceConfigMap(client, rightConfigMapAfterChange, "right");
TestUtil.replaceConfigMap(client, rightConfigMapAfterChange, "right");
// nothing changes in our app, because we are watching only labeled configmaps
await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(30)).until(() -> {
String innerRightResult = rightWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
.retryWhen(retrySpec()).block();
.retryWhen(TestUtil.retrySpec()).block();
return "right-initial".equals(innerRightResult);
});
@@ -234,15 +230,15 @@ class ConfigMapEventReloadIT {
new ObjectMetaBuilder().withNamespace("right").withName("right-configmap-with-label").build())
.withData(Map.of("right.with.label.value", "right-with-label-after-change")).build();
replaceConfigMap(client, rightWithLabelConfigMapAfterChange, "right");
TestUtil.replaceConfigMap(client, rightWithLabelConfigMapAfterChange, "right");
// since we have changed a labeled configmap, app will restart and pick up the new
// value
String[] resultAfterChange = new String[1];
await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> {
WebClient innerWebClient = builder().baseUrl("http://localhost/with-label").build();
WebClient innerWebClient = TestUtil.builder().baseUrl("http://localhost/with-label").build();
String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
.retryWhen(retrySpec()).block();
.retryWhen(TestUtil.retrySpec()).block();
resultAfterChange[0] = innerResult;
return innerResult != null;
});
@@ -250,7 +246,7 @@ class ConfigMapEventReloadIT {
// right-configmap now will see the new value also, but only because the other
// configmap has triggered the restart
rightResult = rightWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec())
rightResult = rightWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(TestUtil.retrySpec())
.block();
Assertions.assertEquals("right-after-change", rightResult);
}
@@ -267,15 +263,15 @@ class ConfigMapEventReloadIT {
*/
void testInformFromOneNamespaceEventTriggeredSecretsDisabled() {
reCreateConfigMaps(util, client);
patchThree(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE);
TestUtil.reCreateConfigMaps(util, client);
TestUtil.patchThree(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE);
Commons.assertReloadLogStatements("added configmap informer for namespace",
"added secret informer for namespace", IMAGE_NAME);
// read the value from the right-configmap
WebClient webClient = builder().baseUrl("http://localhost/right").build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec())
WebClient webClient = TestUtil.builder().baseUrl("http://localhost/right").build();
String result = webClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(TestUtil.retrySpec())
.block();
Assertions.assertEquals("right-initial", result);
@@ -284,13 +280,13 @@ class ConfigMapEventReloadIT {
.withMetadata(new ObjectMetaBuilder().withNamespace("right").withName("right-configmap").build())
.withData(Map.of("right.value", "right-after-change")).build();
replaceConfigMap(client, rightConfigMapAfterChange, "right");
TestUtil.replaceConfigMap(client, rightConfigMapAfterChange, "right");
String[] resultAfterChange = new String[1];
await().pollInterval(Duration.ofSeconds(3)).atMost(Duration.ofSeconds(90)).until(() -> {
WebClient innerWebClient = builder().baseUrl("http://localhost/right").build();
WebClient innerWebClient = TestUtil.builder().baseUrl("http://localhost/right").build();
String innerResult = innerWebClient.method(HttpMethod.GET).retrieve().bodyToMono(String.class)
.retryWhen(retrySpec()).block();
.retryWhen(TestUtil.retrySpec()).block();
resultAfterChange[0] = innerResult;
return innerResult != null;
});
@@ -299,9 +295,27 @@ class ConfigMapEventReloadIT {
}
void testDataChangesInConfigMap() {
reCreateConfigMaps(util, client);
patchFour(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE);
DataChangesInConfigMapReloadDelegate.testDataChangesInConfigMap(client);
TestUtil.reCreateConfigMaps(util, client);
TestUtil.patchFour(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE);
DataChangesInConfigMapReloadDelegate.testDataChangesInConfigMap(client, K3S, IMAGE_NAME);
}
void testConfigMapPollingReload() {
TestUtil.patchFive(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE);
ConfigMapPollingReloadDelegate.testConfigMapPollingReload(client);
}
void testConfigMapMountPollingReload() {
TestUtil.reCreateConfigMaps(util, client);
TestUtil.patchSix(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE);
ConfigMapMountPollingReloadDelegate.testConfigMapMountPollingReload(client, util, K3S, IMAGE_NAME);
}
void testPollingReloadConfigMapWithBootstrap() {
TestUtil.reCreateConfigMaps(util, client);
TestUtil.patchSeven(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE);
BootstrapEnabledPollingReloadConfigMapMountDelegate.testPollingReloadConfigMapWithBootstrap(client, util, K3S,
IMAGE_NAME);
}
private static void manifests(Phase phase) {
@@ -312,6 +326,7 @@ class ConfigMapEventReloadIT {
InputStream leftConfigMapStream = util.inputStream("left-configmap.yaml");
InputStream rightConfigMapStream = util.inputStream("right-configmap.yaml");
InputStream rightWithLabelConfigMapStream = util.inputStream("right-configmap-with-label.yaml");
InputStream configMapAsStream = util.inputStream("configmap.yaml");
Deployment deployment = Serialization.unmarshal(deploymentStream, Deployment.class);
@@ -320,17 +335,20 @@ class ConfigMapEventReloadIT {
ConfigMap leftConfigMap = Serialization.unmarshal(leftConfigMapStream, ConfigMap.class);
ConfigMap rightConfigMap = Serialization.unmarshal(rightConfigMapStream, ConfigMap.class);
ConfigMap rightWithLabelConfigMap = Serialization.unmarshal(rightWithLabelConfigMapStream, ConfigMap.class);
ConfigMap configMap = Serialization.unmarshal(configMapAsStream, ConfigMap.class);
if (phase.equals(Phase.CREATE)) {
util.createAndWait("left", leftConfigMap, null);
util.createAndWait("right", rightConfigMap, null);
util.createAndWait("right", rightWithLabelConfigMap, null);
util.createAndWait(NAMESPACE, configMap, null);
util.createAndWait(NAMESPACE, null, deployment, service, ingress, true);
}
else {
util.deleteAndWait("left", leftConfigMap, null);
util.deleteAndWait("right", rightConfigMap, null);
util.deleteAndWait("right", rightWithLabelConfigMap, null);
util.deleteAndWait(NAMESPACE, configMap, null);
util.deleteAndWait(NAMESPACE, deployment, service, ingress);
}

View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.fabric8.configmap.event.reload;
package org.springframework.cloud.kubernetes.fabric8.reload;
import java.io.InputStream;
import java.time.Duration;
@@ -24,6 +24,8 @@ import java.util.Objects;
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.utils.Serialization;
import org.testcontainers.containers.Container;
import org.testcontainers.k3s.K3sContainer;
import reactor.netty.http.client.HttpClient;
import reactor.util.retry.Retry;
import reactor.util.retry.RetryBackoffSpec;
@@ -38,7 +40,7 @@ import org.springframework.web.reactive.function.client.WebClient;
final class TestUtil {
private static final Map<String, String> POD_LABELS = Map.of("app",
"spring-cloud-kubernetes-fabric8-client-configmap-event-reload");
"spring-cloud-kubernetes-fabric8-client-reload");
private static final String BODY_ONE = """
{
@@ -56,6 +58,10 @@ final class TestUtil {
{
"name": "SPRING_PROFILES_ACTIVE",
"value": "two"
},
{
"name": "SPRING_CLOUD_BOOTSTRAP_ENABLED",
"value": "TRUE"
}
]
}]
@@ -81,6 +87,10 @@ final class TestUtil {
{
"name": "SPRING_PROFILES_ACTIVE",
"value": "three"
},
{
"name": "SPRING_CLOUD_BOOTSTRAP_ENABLED",
"value": "TRUE"
}
]
}]
@@ -110,6 +120,10 @@ final class TestUtil {
{
"name": "SPRING_CLOUD_KUBERNETES_SECRETS_ENABLED",
"value": "FALSE"
},
{
"name": "SPRING_CLOUD_BOOTSTRAP_ENABLED",
"value": "TRUE"
}
]
}]
@@ -143,6 +157,139 @@ final class TestUtil {
{
"name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_CLIENT_CONFIG_RELOAD",
"value": "DEBUG"
},
{
"name": "SPRING_CLOUD_BOOTSTRAP_ENABLED",
"value": "TRUE"
}
]
}]
}
}
}
}
""";
private static final String BODY_FIVE = """
{
"spec": {
"template": {
"spec": {
"containers": [{
"name": "spring-cloud-kubernetes-fabric8-client-configmap-event-reload",
"image": "image_name_here",
"env": [
{
"name": "SPRING_PROFILES_ACTIVE",
"value": "no-mount"
},
{
"name": "SPRING_CLOUD_BOOTSTRAP_ENABLED",
"value": "FALSE"
}
]
}]
}
}
}
}
""";
private static final String BODY_SIX = """
{
"spec": {
"template": {
"spec": {
"volumes": [
{
"configMap": {
"defaultMode": 420,
"name": "poll-reload"
},
"name": "config-map-volume"
}
],
"containers": [{
"volumeMounts": [
{
"mountPath": "/tmp",
"name": "config-map-volume"
}
],
"name": "spring-cloud-kubernetes-fabric8-client-configmap-event-reload",
"image": "image_name_here",
"env": [
{
"name": "SPRING_PROFILES_ACTIVE",
"value": "mount"
},
{
"name": "SPRING_CLOUD_BOOTSTRAP_ENABLED",
"value": "FALSE"
},
{
"name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_COMMONS_CONFIG_RELOAD",
"value": "DEBUG"
},
{
"name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_COMMONS_CONFIG",
"value": "DEBUG"
},
{
"name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_COMMONS",
"value": "DEBUG"
}
]
}]
}
}
}
}
""";
private static final String BODY_SEVEN = """
{
"spec": {
"template": {
"spec": {
"volumes": [
{
"configMap": {
"defaultMode": 420,
"name": "poll-reload"
},
"name": "config-map-volume"
}
],
"containers": [{
"volumeMounts": [
{
"mountPath": "/tmp",
"name": "config-map-volume"
}
],
"name": "spring-cloud-kubernetes-fabric8-client-configmap-event-reload",
"image": "image_name_here",
"env": [
{
"name": "SPRING_PROFILES_ACTIVE",
"value": "with-bootstrap"
},
{
"name": "SPRING_CLOUD_BOOTSTRAP_ENABLED",
"value": "TRUE"
},
{
"name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_COMMONS_CONFIG_RELOAD",
"value": "DEBUG"
},
{
"name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_COMMONS_CONFIG",
"value": "DEBUG"
},
{
"name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_COMMONS",
"value": "DEBUG"
}
]
}]
@@ -159,12 +306,15 @@ final class TestUtil {
static void reCreateConfigMaps(Util util, KubernetesClient client) {
InputStream leftConfigMapStream = util.inputStream("left-configmap.yaml");
InputStream rightConfigMapStream = util.inputStream("right-configmap.yaml");
InputStream configMapStream = util.inputStream("configmap.yaml");
ConfigMap leftConfigMap = Serialization.unmarshal(leftConfigMapStream, ConfigMap.class);
ConfigMap rightConfigMap = Serialization.unmarshal(rightConfigMapStream, ConfigMap.class);
ConfigMap configMap = Serialization.unmarshal(configMapStream, ConfigMap.class);
replaceConfigMap(client, leftConfigMap, "left");
replaceConfigMap(client, rightConfigMap, "right");
replaceConfigMap(client, configMap, "default");
}
static void replaceConfigMap(KubernetesClient client, ConfigMap configMap, String namespace) {
@@ -187,6 +337,18 @@ final class TestUtil {
util.patchWithReplace(dockerImage, deploymentName, namespace, BODY_FOUR, POD_LABELS);
}
static void patchFive(Util util, String dockerImage, String deploymentName, String namespace) {
util.patchWithReplace(dockerImage, deploymentName, namespace, BODY_FIVE, POD_LABELS);
}
static void patchSix(Util util, String dockerImage, String deploymentName, String namespace) {
util.patchWithReplace(dockerImage, deploymentName, namespace, BODY_SIX, POD_LABELS);
}
static void patchSeven(Util util, String dockerImage, String deploymentName, String namespace) {
util.patchWithReplace(dockerImage, deploymentName, namespace, BODY_SEVEN, POD_LABELS);
}
static WebClient.Builder builder() {
return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create()));
}
@@ -195,4 +357,20 @@ final class TestUtil {
return Retry.fixedDelay(120, Duration.ofSeconds(2)).filter(Objects::nonNull);
}
static String logs(K3sContainer container, String appLabelValue) {
try {
String appPodName = container
.execInContainer("sh", "-c",
"kubectl get pods -l app=" + appLabelValue + " -o=name --no-headers | tr -d '\n'")
.getStdout();
Container.ExecResult execResult = container.execInContainer("sh", "-c",
"kubectl logs " + appPodName.trim());
return execResult.getStdout();
}
catch (Exception e) {
throw new RuntimeException(e);
}
}
}

View File

@@ -1,20 +1,20 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: spring-cloud-kubernetes-fabric8-client-configmap-event-reload
name: spring-cloud-kubernetes-fabric8-client-reload
spec:
selector:
matchLabels:
app: spring-cloud-kubernetes-fabric8-client-configmap-event-reload
app: spring-cloud-kubernetes-fabric8-client-reload
template:
metadata:
labels:
app: spring-cloud-kubernetes-fabric8-client-configmap-event-reload
app: spring-cloud-kubernetes-fabric8-client-reload
spec:
serviceAccountName: spring-cloud-kubernetes-serviceaccount
containers:
- name: spring-cloud-kubernetes-fabric8-client-configmap-event-reload
image: docker.io/springcloud/spring-cloud-kubernetes-fabric8-client-configmap-event-reload
- name: spring-cloud-kubernetes-fabric8-client-reload
image: docker.io/springcloud/spring-cloud-kubernetes-fabric8-client-reload
imagePullPolicy: IfNotPresent
readinessProbe:
httpGet:
@@ -31,3 +31,5 @@ spec:
value: "DEBUG"
- name: SPRING_PROFILES_ACTIVE
value: "one"
- name: SPRING_CLOUD_BOOTSTRAP_ENABLED
value: "TRUE"

View File

@@ -1,7 +1,7 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: spring-cloud-kubernetes-fabric8-client-configmap-ingress-event-reload
name: spring-cloud-kubernetes-fabric8-client-reload-ingress
namespace: default
spec:
rules:
@@ -11,6 +11,6 @@ spec:
pathType: Prefix
backend:
service:
name: spring-cloud-kubernetes-fabric8-client-configmap-event-reload
name: spring-cloud-kubernetes-fabric8-client-reload
port:
number: 8080

View File

@@ -6,4 +6,4 @@ metadata:
labels:
spring.cloud.kubernetes.config.informer.enabled: true
data:
right.with.label.value: right-with-label-initial
right.with.label.value: "right-with-label-initial"

View File

@@ -0,0 +1,14 @@
apiVersion: v1
kind: Service
metadata:
labels:
app: spring-cloud-kubernetes-fabric8-client-reload
name: spring-cloud-kubernetes-fabric8-client-reload
spec:
ports:
- name: http
port: 8080
targetPort: 8080
selector:
app: spring-cloud-kubernetes-fabric8-client-reload
type: ClusterIP

View File

@@ -116,7 +116,7 @@ public final class Commons {
LOG.info("appPodName : ->" + appPodName + "<-");
// we issue a pollDelay to let the logs sync in, otherwise the results are not
// going to be correctly asserted
await().pollDelay(20, TimeUnit.SECONDS).pollInterval(Duration.ofSeconds(5)).atMost(Duration.ofSeconds(600))
await().pollDelay(20, TimeUnit.SECONDS).pollInterval(Duration.ofSeconds(5)).atMost(Duration.ofSeconds(120))
.until(() -> {
Container.ExecResult result = CONTAINER.execInContainer("sh", "-c",