diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java index 9a7522ca..0356996a 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedConfigMapChangeDetector.java @@ -24,6 +24,7 @@ import io.kubernetes.client.common.KubernetesObject; import io.kubernetes.client.informer.ResourceEventHandler; import io.kubernetes.client.informer.SharedIndexInformer; import io.kubernetes.client.informer.SharedInformerFactory; +import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1ConfigMap; import io.kubernetes.client.openapi.models.V1ConfigMapList; @@ -57,10 +58,12 @@ public class KubernetesClientEventBasedConfigMapChangeDetector extends Configura private final KubernetesClientConfigMapPropertySourceLocator propertySourceLocator; - private final SharedInformerFactory factory; + private final ApiClient apiClient; private final List> informers = new ArrayList<>(); + private final List factories = new ArrayList<>(); + private final Set namespaces; private final boolean enableReloadFiltering; @@ -69,19 +72,22 @@ public class KubernetesClientEventBasedConfigMapChangeDetector extends Configura @Override public void onAdd(V1ConfigMap configMap) { - LOG.debug(() -> "ConfigMap " + configMap.getMetadata().getName() + " was added."); + LOG.debug(() -> "ConfigMap " + configMap.getMetadata().getName() + " was added in namespace " + + configMap.getMetadata().getNamespace()); onEvent(configMap); } @Override public void onUpdate(V1ConfigMap oldConfigMap, V1ConfigMap newConfigMap) { - LOG.debug(() -> "ConfigMap " + newConfigMap.getMetadata().getName() + " was updated."); + LOG.debug(() -> "ConfigMap " + newConfigMap.getMetadata().getName() + " was updated in namespace " + + newConfigMap.getMetadata().getNamespace()); onEvent(newConfigMap); } @Override public void onDelete(V1ConfigMap configMap, boolean deletedFinalStateUnknown) { - LOG.debug(() -> "ConfigMap " + configMap.getMetadata().getName() + " was deleted."); + LOG.debug(() -> "ConfigMap " + configMap.getMetadata().getName() + " was deleted in namespace " + + configMap.getMetadata().getNamespace()); onEvent(configMap); } }; @@ -93,14 +99,7 @@ public class KubernetesClientEventBasedConfigMapChangeDetector extends Configura super(environment, properties, strategy); this.propertySourceLocator = propertySourceLocator; this.coreV1Api = coreV1Api; - // We need to pass an APIClient to the SharedInformerFactory because if we use the - // default - // constructor it will use the configured default APIClient but that may not - // contain - // an APIClient configured within the cluster and does not contain the necessary - // certificate authorities for the cluster. This results in SSL errors. - // See https://github.com/spring-cloud/spring-cloud-kubernetes/issues/885 - this.factory = new SharedInformerFactory(createApiClientForInformerClient()); + this.apiClient = createApiClientForInformerClient(); this.enableReloadFiltering = properties.enableReloadFiltering(); namespaces = namespaces(kubernetesNamespaceProvider, properties, "configmap"); } @@ -111,34 +110,42 @@ public class KubernetesClientEventBasedConfigMapChangeDetector extends Configura namespaces.forEach(namespace -> { SharedIndexInformer informer; - String filter = null; + String[] filter = new String[1]; if (enableReloadFiltering) { - filter = ConfigReloadProperties.RELOAD_LABEL_FILTER + "=true"; - LOG.debug(() -> "added configmap informer for namespace : " + namespace + " with enabled filter"); - } - else { - LOG.debug(() -> "added configmap informer for namespace : " + namespace); + filter[0] = ConfigReloadProperties.RELOAD_LABEL_FILTER + "=true"; } - String filterOnInformerLabel = filter; - informer = factory - .sharedIndexInformerFor( - (CallGeneratorParams params) -> coreV1Api.listNamespacedConfigMapCall(namespace, null, null, - null, null, filterOnInformerLabel, null, params.resourceVersion, null, - params.timeoutSeconds, params.watch, null), - V1ConfigMap.class, V1ConfigMapList.class); + // We need to pass an APIClient to the SharedInformerFactory because if we use + // the + // default + // constructor it will use the configured default APIClient but that may not + // contain + // an APIClient configured within the cluster and does not contain the + // necessary + // certificate authorities for the cluster. This results in SSL errors. + // See https://github.com/spring-cloud/spring-cloud-kubernetes/issues/885 + SharedInformerFactory factory = new SharedInformerFactory(apiClient); + factories.add(factory); + informer = factory.sharedIndexInformerFor( + (CallGeneratorParams params) -> coreV1Api.listNamespacedConfigMapCall(namespace, null, null, null, + null, filter[0], null, params.resourceVersion, null, params.timeoutSeconds, params.watch, + null), + V1ConfigMap.class, V1ConfigMapList.class); + + LOG.debug(() -> "added configmap informer for namespace : " + namespace + " with filter : " + filter[0]); + informer.addEventHandler(handler); informers.add(informer); + factory.startAllRegisteredInformers(); }); - factory.startAllRegisteredInformers(); } @PreDestroy void shutdown() { informers.forEach(SharedIndexInformer::stop); - factory.stopAllRegisteredInformers(); + factories.forEach(SharedInformerFactory::stopAllRegisteredInformers); } protected void onEvent(KubernetesObject configMap) { diff --git a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java index c6a5dbf0..1f22e967 100644 --- a/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java +++ b/spring-cloud-kubernetes-client-config/src/main/java/org/springframework/cloud/kubernetes/client/config/reload/KubernetesClientEventBasedSecretsChangeDetector.java @@ -24,6 +24,7 @@ import io.kubernetes.client.common.KubernetesObject; import io.kubernetes.client.informer.ResourceEventHandler; import io.kubernetes.client.informer.SharedIndexInformer; import io.kubernetes.client.informer.SharedInformerFactory; +import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1Secret; import io.kubernetes.client.openapi.models.V1SecretList; @@ -57,10 +58,12 @@ public class KubernetesClientEventBasedSecretsChangeDetector extends Configurati private final KubernetesClientSecretsPropertySourceLocator propertySourceLocator; - private final SharedInformerFactory factory; + private final ApiClient apiClient; private final List> informers = new ArrayList<>(); + private final List factories = new ArrayList<>(); + private final Set namespaces; private final boolean enableReloadFiltering; @@ -69,19 +72,22 @@ public class KubernetesClientEventBasedSecretsChangeDetector extends Configurati @Override public void onAdd(V1Secret secret) { - LOG.debug(() -> "Secret " + secret.getMetadata().getName() + " was added."); + LOG.debug(() -> "Secret " + secret.getMetadata().getName() + " was added in namespace " + + secret.getMetadata().getNamespace()); onEvent(secret); } @Override public void onUpdate(V1Secret oldSecret, V1Secret newSecret) { - LOG.debug(() -> "Secret " + newSecret.getMetadata().getName() + " was updated."); + LOG.debug(() -> "Secret " + newSecret.getMetadata().getName() + " was updated in namespace " + + newSecret.getMetadata().getNamespace()); onEvent(newSecret); } @Override public void onDelete(V1Secret secret, boolean deletedFinalStateUnknown) { - LOG.debug(() -> "Secret " + secret.getMetadata().getName() + " was deleted."); + LOG.debug(() -> "Secret " + secret.getMetadata().getName() + " was deleted in namespace " + + secret.getMetadata().getNamespace()); onEvent(secret); } }; @@ -93,14 +99,7 @@ public class KubernetesClientEventBasedSecretsChangeDetector extends Configurati super(environment, properties, strategy); this.propertySourceLocator = propertySourceLocator; this.coreV1Api = coreV1Api; - // We need to pass an APIClient to the SharedInformerFactory because if we use the - // default - // constructor it will use the configured default APIClient but that may not - // contain - // an APIClient configured within the cluster and does not contain the necessary - // certificate authorities for the cluster. This results in SSL errors. - // See https://github.com/spring-cloud/spring-cloud-kubernetes/issues/885 - this.factory = new SharedInformerFactory(createApiClientForInformerClient()); + this.apiClient = createApiClientForInformerClient(); this.enableReloadFiltering = properties.enableReloadFiltering(); namespaces = namespaces(kubernetesNamespaceProvider, properties, "secret"); } @@ -111,33 +110,42 @@ public class KubernetesClientEventBasedSecretsChangeDetector extends Configurati namespaces.forEach(namespace -> { SharedIndexInformer informer; - String filter = null; + String[] filter = new String[1]; if (enableReloadFiltering) { - filter = ConfigReloadProperties.RELOAD_LABEL_FILTER + "=true"; - LOG.debug(() -> "added secret informer for namespace : " + namespace + " with enabled filter"); - } - else { - LOG.debug(() -> "added secret informer for namespace : " + namespace); + filter[0] = ConfigReloadProperties.RELOAD_LABEL_FILTER + "=true"; } - String filterOnInformerLabel = filter; + // We need to pass an APIClient to the SharedInformerFactory because if we use + // the + // default + // constructor it will use the configured default APIClient but that may not + // contain + // an APIClient configured within the cluster and does not contain the + // necessary + // certificate authorities for the cluster. This results in SSL errors. + // See https://github.com/spring-cloud/spring-cloud-kubernetes/issues/885 + SharedInformerFactory factory = new SharedInformerFactory(apiClient); + factories.add(factory); informer = factory.sharedIndexInformerFor( (CallGeneratorParams params) -> coreV1Api.listNamespacedSecretCall(namespace, null, null, null, - null, filterOnInformerLabel, null, params.resourceVersion, null, params.timeoutSeconds, - params.watch, null), + null, filter[0], null, params.resourceVersion, null, params.timeoutSeconds, params.watch, + null), V1Secret.class, V1SecretList.class); + + LOG.debug(() -> "added secret informer for namespace : " + namespace + " with filter : " + filter[0]); + informer.addEventHandler(handler); informers.add(informer); + factory.startAllRegisteredInformers(); }); - factory.startAllRegisteredInformers(); } @PreDestroy void shutdown() { informers.forEach(SharedIndexInformer::stop); - factory.stopAllRegisteredInformers(); + factories.forEach(SharedInformerFactory::stopAllRegisteredInformers); } protected void onEvent(KubernetesObject secret) { diff --git a/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedConfigMapChangeDetector.java b/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedConfigMapChangeDetector.java index f8a98fdc..fdb91e98 100644 --- a/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedConfigMapChangeDetector.java +++ b/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedConfigMapChangeDetector.java @@ -122,19 +122,22 @@ public class Fabric8EventBasedConfigMapChangeDetector extends ConfigurationChang @Override public void onAdd(ConfigMap configMap) { - LOG.debug("ConfigMap " + configMap.getMetadata().getName() + " was added."); + LOG.debug("ConfigMap " + configMap.getMetadata().getName() + " was added in namespace " + + configMap.getMetadata().getNamespace()); onEvent(configMap); } @Override public void onUpdate(ConfigMap oldConfigMap, ConfigMap newConfigMap) { - LOG.debug("ConfigMap " + newConfigMap.getMetadata().getName() + " was updated."); + LOG.debug("ConfigMap " + newConfigMap.getMetadata().getName() + " was updated in namespace " + + newConfigMap.getMetadata().getNamespace()); onEvent(newConfigMap); } @Override public void onDelete(ConfigMap configMap, boolean deletedFinalStateUnknown) { - LOG.debug("ConfigMap " + configMap.getMetadata().getName() + " was deleted."); + LOG.debug("ConfigMap " + configMap.getMetadata().getName() + " was deleted in namespace " + + configMap.getMetadata().getName()); onEvent(configMap); } diff --git a/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedSecretsChangeDetector.java b/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedSecretsChangeDetector.java index dca6e192..74615e57 100644 --- a/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedSecretsChangeDetector.java +++ b/spring-cloud-kubernetes-fabric8-config/src/main/java/org/springframework/cloud/kubernetes/fabric8/config/reload/Fabric8EventBasedSecretsChangeDetector.java @@ -124,19 +124,22 @@ public class Fabric8EventBasedSecretsChangeDetector extends ConfigurationChangeD @Override public void onAdd(Secret secret) { - LOG.debug("Secret " + secret.getMetadata().getName() + " was added."); + LOG.debug("Secret " + secret.getMetadata().getName() + " was added in namespace " + + secret.getMetadata().getNamespace()); onEvent(secret); } @Override public void onUpdate(Secret oldSecret, Secret newSecret) { - LOG.debug("Secret " + newSecret.getMetadata().getName() + " was updated."); + LOG.debug("Secret " + newSecret.getMetadata().getName() + " was updated in namespace " + + newSecret.getMetadata().getNamespace()); onEvent(newSecret); } @Override public void onDelete(Secret secret, boolean deletedFinalStateUnknown) { - LOG.debug("Secret " + secret.getMetadata().getName() + " was deleted."); + LOG.debug("Secret " + secret.getMetadata().getName() + " was deleted in namespace " + + secret.getMetadata().getNamespace()); onEvent(secret); } diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-configuration-watcher-it/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ActuatorRefreshIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-configuration-watcher-it/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ActuatorRefreshIT.java index 84bfb6d0..7bbffb6e 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-configuration-watcher-it/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ActuatorRefreshIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-configuration-watcher-it/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ActuatorRefreshIT.java @@ -18,7 +18,6 @@ package org.springframework.cloud.kubernetes.configuration.watcher; import java.time.Duration; -import com.github.tomakehurst.wiremock.client.VerificationException; import com.github.tomakehurst.wiremock.client.WireMock; import io.kubernetes.client.openapi.models.V1ConfigMap; import io.kubernetes.client.openapi.models.V1ConfigMapBuilder; @@ -93,7 +92,7 @@ class ActuatorRefreshIT { @Test void testActuatorRefresh() { WireMock.configureFor(WIREMOCK_HOST, WIREMOCK_PORT, WIREMOCK_PATH); - await().timeout(Duration.ofSeconds(60)).ignoreException(VerificationException.class) + await().timeout(Duration.ofSeconds(60)) .until(() -> WireMock .stubFor(WireMock.post(WireMock.urlEqualTo("/actuator/refresh")) .willReturn(WireMock.aResponse().withBody("{}").withStatus(200))) diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-configuration-watcher-it/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ActuatorRefreshMultipleNamespacesIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-configuration-watcher-it/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ActuatorRefreshMultipleNamespacesIT.java new file mode 100644 index 00000000..808f747e --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-configuration-watcher-it/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/ActuatorRefreshMultipleNamespacesIT.java @@ -0,0 +1,221 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.github.tomakehurst.wiremock.client.WireMock; +import io.kubernetes.client.openapi.models.V1ConfigMap; +import io.kubernetes.client.openapi.models.V1ConfigMapBuilder; +import io.kubernetes.client.openapi.models.V1Deployment; +import io.kubernetes.client.openapi.models.V1EnvVar; +import io.kubernetes.client.openapi.models.V1Secret; +import io.kubernetes.client.openapi.models.V1SecretBuilder; +import io.kubernetes.client.openapi.models.V1Service; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.k3s.K3sContainer; + +import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; +import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; +import org.springframework.cloud.kubernetes.integration.tests.commons.native_client.Util; + +import static org.awaitility.Awaitility.await; + +class ActuatorRefreshMultipleNamespacesIT { + + private static final String SPRING_CLOUD_K8S_CONFIG_WATCHER_APP_NAME = "spring-cloud-kubernetes-configuration-watcher"; + + private static final String WIREMOCK_HOST = "localhost"; + + private static final String WIREMOCK_PATH = "/"; + + private static final int WIREMOCK_PORT = 80; + + private static final String DEFAULT_NAMESPACE = "default"; + + private static final String LEFT_NAMESPACE = "left"; + + private static final String RIGHT_NAMESPACE = "right"; + + private static final K3sContainer K3S = Commons.container(); + + private static Util util; + + @BeforeAll + static void beforeAll() throws Exception { + K3S.start(); + Commons.validateImage(SPRING_CLOUD_K8S_CONFIG_WATCHER_APP_NAME, K3S); + Commons.loadSpringCloudKubernetesImage(SPRING_CLOUD_K8S_CONFIG_WATCHER_APP_NAME, K3S); + util = new Util(K3S); + util.createNamespace(LEFT_NAMESPACE); + util.createNamespace(RIGHT_NAMESPACE); + util.setUpClusterWide(DEFAULT_NAMESPACE, Set.of(DEFAULT_NAMESPACE, LEFT_NAMESPACE, RIGHT_NAMESPACE)); + } + + @AfterAll + static void afterAll() throws Exception { + util.deleteNamespace(LEFT_NAMESPACE); + util.deleteNamespace(RIGHT_NAMESPACE); + Commons.cleanUp(SPRING_CLOUD_K8S_CONFIG_WATCHER_APP_NAME, K3S); + } + + @BeforeEach + void setup() { + configWatcher(Phase.CREATE); + util.wiremock(DEFAULT_NAMESPACE, "/", Phase.CREATE); + } + + @AfterEach + void after() { + configWatcher(Phase.DELETE); + util.wiremock(DEFAULT_NAMESPACE, "/", Phase.DELETE); + } + + /** + * 
+	 *     - deploy config-watcher in default namespace
+	 *     - deploy wiremock in default namespace (so that we could assert calls to the actuator path)
+	 *     - deploy configmap-left in left namespaces with proper label and "service-wiremock" name. Because of the
+	 *       label, this will trigger a reload; because of the name this will trigger a reload against that name.
+	 *       This is a http refresh against the actuator.
+	 *     - same as above for the configmap-right.
+	 *
+ */ + @Test + void testConfigMapActuatorRefreshMultipleNamespaces() { + WireMock.configureFor(WIREMOCK_HOST, WIREMOCK_PORT, WIREMOCK_PATH); + await().timeout(Duration.ofSeconds(60)) + .until(() -> WireMock + .stubFor(WireMock.post(WireMock.urlEqualTo("/actuator/refresh")) + .willReturn(WireMock.aResponse().withBody("{}").withStatus(200))) + .getResponse().wasConfigured()); + + // left-config-map + V1ConfigMap leftConfigMap = new V1ConfigMapBuilder().editOrNewMetadata() + .withLabels(Map.of("spring.cloud.kubernetes.config", "true")).withName("service-wiremock") + .withNamespace(LEFT_NAMESPACE).endMetadata().addToData("color", "purple").build(); + util.createAndWait(LEFT_NAMESPACE, leftConfigMap, null); + + // right-config-map + V1ConfigMap rightConfigMap = new V1ConfigMapBuilder().editOrNewMetadata() + .withLabels(Map.of("spring.cloud.kubernetes.config", "true")).withName("service-wiremock") + .withNamespace(RIGHT_NAMESPACE).endMetadata().addToData("color", "green").build(); + util.createAndWait(RIGHT_NAMESPACE, rightConfigMap, null); + + // comes from handler::onAdd (and as such from "onEvent") + Commons.assertReloadLogStatements("ConfigMap service-wiremock was added in namespace left", "", + "spring-cloud-kubernetes-configuration-watcher"); + + // comes from handler::onAdd (and as such from "onEvent") + Commons.assertReloadLogStatements("ConfigMap service-wiremock was added in namespace right", "", + "spring-cloud-kubernetes-configuration-watcher"); + + await().atMost(Duration.ofSeconds(30)).until( + () -> !WireMock.findAll(WireMock.postRequestedFor(WireMock.urlEqualTo("/actuator/refresh"))).isEmpty()); + WireMock.verify(WireMock.exactly(2), WireMock.postRequestedFor(WireMock.urlEqualTo("/actuator/refresh"))); + + util.deleteAndWait(LEFT_NAMESPACE, leftConfigMap, null); + util.deleteAndWait(RIGHT_NAMESPACE, rightConfigMap, null); + } + + /** + * 
+	 *     - deploy config-watcher in default namespace
+	 *     - deploy wiremock in default namespace (so that we could assert calls to the actuator path)
+	 *     - deploy secret-left in left namespaces with proper label and "service-wiremock". Because of the
+	 *       label, this will trigger a reload; because of the name this will trigger a reload against that name.
+	 *       This is a http refresh against the actuator.
+	 *     - same as above for the secret-right.
+	 *
+ */ + @Test + void testSecretActuatorRefreshMultipleNamespaces() { + WireMock.configureFor(WIREMOCK_HOST, WIREMOCK_PORT, WIREMOCK_PATH); + await().timeout(Duration.ofSeconds(60)) + .until(() -> WireMock + .stubFor(WireMock.post(WireMock.urlEqualTo("/actuator/refresh")) + .willReturn(WireMock.aResponse().withBody("{}").withStatus(200))) + .getResponse().wasConfigured()); + + // left-secret + V1Secret leftSecret = new V1SecretBuilder().editOrNewMetadata() + .withLabels(Map.of("spring.cloud.kubernetes.secret", "true")).withName("service-wiremock") + .withNamespace(LEFT_NAMESPACE).endMetadata() + .addToData("color", Base64.getEncoder().encode("purple".getBytes(StandardCharsets.UTF_8))).build(); + util.createAndWait(LEFT_NAMESPACE, null, leftSecret); + + // right-secret + V1Secret rightSecret = new V1SecretBuilder().editOrNewMetadata() + .withLabels(Map.of("spring.cloud.kubernetes.secret", "true")).withName("service-wiremock") + .withNamespace(RIGHT_NAMESPACE).endMetadata() + .addToData("color", Base64.getEncoder().encode("green".getBytes(StandardCharsets.UTF_8))).build(); + util.createAndWait(RIGHT_NAMESPACE, null, rightSecret); + + // comes from handler::onAdd (and as such from "onEvent") + Commons.assertReloadLogStatements("Secret service-wiremock was added in namespace left", "", + "spring-cloud-kubernetes-configuration-watcher"); + + // comes from handler::onAdd (and as such from "onEvent") + Commons.assertReloadLogStatements("Secret service-wiremock was added in namespace right", "", + "spring-cloud-kubernetes-configuration-watcher"); + + await().atMost(Duration.ofSeconds(30)).until( + () -> !WireMock.findAll(WireMock.postRequestedFor(WireMock.urlEqualTo("/actuator/refresh"))).isEmpty()); + WireMock.verify(WireMock.exactly(2), WireMock.postRequestedFor(WireMock.urlEqualTo("/actuator/refresh"))); + + util.deleteAndWait(LEFT_NAMESPACE, null, leftSecret); + util.deleteAndWait(RIGHT_NAMESPACE, null, rightSecret); + + } + + private void configWatcher(Phase phase) { + V1ConfigMap configMap = (V1ConfigMap) util + .yaml("config-watcher/spring-cloud-kubernetes-configuration-watcher-configmap.yaml"); + V1Deployment deployment = (V1Deployment) util + .yaml("config-watcher/spring-cloud-kubernetes-configuration-watcher-http-deployment.yaml"); + + List envVars = List.of( + new V1EnvVar().name("SPRING_CLOUD_KUBERNETES_RELOAD_NAMESPACES_0").value(LEFT_NAMESPACE), + new V1EnvVar().name("SPRING_CLOUD_KUBERNETES_RELOAD_NAMESPACES_1").value(RIGHT_NAMESPACE), + new V1EnvVar().name("LOGGING_LEVEL_ORG_SPRINGFRAMEWORK").value("TRACE")); + + deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars); + + V1Service service = (V1Service) util + .yaml("config-watcher/spring-cloud-kubernetes-configuration-watcher-service.yaml"); + + if (phase.equals(Phase.CREATE)) { + util.createAndWait(DEFAULT_NAMESPACE, configMap, null); + util.createAndWait(DEFAULT_NAMESPACE, null, deployment, service, null, true); + } + else { + util.deleteAndWait(DEFAULT_NAMESPACE, configMap, null); + util.deleteAndWait(DEFAULT_NAMESPACE, deployment, service, null); + } + + } + +} diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java index d1418dc1..dd737834 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java @@ -26,6 +26,7 @@ import java.nio.file.Paths; import java.time.Duration; import java.util.Arrays; import java.util.List; +import java.util.concurrent.TimeUnit; import com.github.dockerjava.api.command.ListImagesCmd; import com.github.dockerjava.api.command.PullImageCmd; @@ -100,22 +101,36 @@ public final class Commons { public static void assertReloadLogStatements(String left, String right, String appLabel) { try { - String appPodName = CONTAINER - .execInContainer("kubectl", "get", "pods", "-l", "app=" + appLabel, "-o=name", "--no-headers") - .getStdout(); - await().pollInterval(Duration.ofSeconds(5)).atMost(Duration.ofSeconds(180)).until(() -> { + String appPodName = CONTAINER.execInContainer("sh", "-c", + "kubectl get pods -l app=" + appLabel + " -o=name --no-headers | tr -d '\n'").getStdout(); + LOG.info("appPodName : ->" + appPodName + "<-"); + // we issue a pollDelay to let the logs sync in, otherwise the results are not + // going to be correctly asserted + await().pollDelay(20, TimeUnit.SECONDS).pollInterval(Duration.ofSeconds(5)).atMost(Duration.ofSeconds(600)) + .until(() -> { - String allLogs = CONTAINER.execInContainer("kubectl", "logs", appPodName.trim()).getStdout(); - LOG.info("=========================================================================================="); - LOG.info(allLogs); - LOG.info("=========================================================================================="); - if (allLogs.contains(left)) { - Assertions.assertFalse(allLogs.contains(right)); - return true; - } - LOG.info("log statement not yet present"); - return false; - }); + Container.ExecResult result = CONTAINER.execInContainer("sh", "-c", + "kubectl logs " + appPodName.trim() + "| grep " + "'" + left + "'"); + String error = result.getStderr(); + String ok = result.getStdout(); + + LOG.info("error is : -->" + error + "<--"); + + if (ok != null && !ok.isBlank()) { + + if (!right.isBlank()) { + String notPresent = CONTAINER + .execInContainer("sh", "-c", + "kubectl logs " + appPodName.trim() + "| grep " + "'" + right + "'") + .getStdout(); + Assertions.assertTrue(notPresent == null || notPresent.isBlank()); + } + + return true; + } + LOG.info("log statement not yet present"); + return false; + }); } catch (Exception e) { throw new RuntimeException(e); diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/Util.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/Util.java index 326bbdad..aefe2cd1 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/Util.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/native_client/Util.java @@ -23,6 +23,7 @@ import java.io.StringReader; import java.net.HttpURLConnection; import java.time.Duration; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -187,9 +188,16 @@ public final class Util { String deploymentName = deploymentName(deployment); String serviceName = serviceName(service); try { + + Map podLabels = appsV1Api.readNamespacedDeployment(deploymentName, namespace, null) + .getSpec().getTemplate().getMetadata().getLabels(); + appsV1Api.deleteNamespacedDeployment(deploymentName, namespace, null, null, null, null, null, null); coreV1Api.deleteNamespacedService(serviceName, namespace, null, null, null, null, null, null); + coreV1Api.deleteCollectionNamespacedPod(namespace, null, null, null, null, null, labelSelector(podLabels), + null, null, null, null, null, null, null); waitForDeploymentToBeDeleted(deploymentName, namespace); + waitForDeploymentPodsToBeDeleted(podLabels, namespace); if (ingress != null) { String ingressName = ingressName(ingress); @@ -465,6 +473,22 @@ public final class Util { }); } + private void waitForDeploymentPodsToBeDeleted(Map labels, String namespace) { + await().timeout(Duration.ofSeconds(90)).until(() -> { + try { + int currentNumberOfPods = coreV1Api.listNamespacedPod(namespace, null, null, null, null, + labelSelector(labels), null, null, null, null, null).getItems().size(); + return currentNumberOfPods == 0; + } + catch (ApiException e) { + if (e.getCode() == HttpURLConnection.HTTP_NOT_FOUND) { + return true; + } + throw new RuntimeException(e); + } + }); + } + private void waitForIngressToBeDeleted(String ingressName, String namespace) { await().timeout(Duration.ofSeconds(90)).until(() -> { try { @@ -508,6 +532,10 @@ public final class Util { } } + private static String labelSelector(Map labels) { + return labels.entrySet().stream().map(en -> en.getKey() + "=" + en.getValue()).collect(Collectors.joining(",")); + } + private interface CheckedSupplier { T get() throws Exception;