From 65c7644f66da4d1a8a3e8d61603cc0bb5ca90bc0 Mon Sep 17 00:00:00 2001 From: erabii Date: Mon, 2 Oct 2023 21:04:34 +0300 Subject: [PATCH] refactor fabric8 catalog watch integration test using PATCH (#1450) --- .../catalog/watch/Fabric8CatalogWatchIT.java | 107 +++++--- .../watch/Fabric8CatalogWatchUtil.java | 156 +++++++++++ ...ic8CatalogWatchWithNamespacesDelegate.java | 146 ++++++++++ .../Fabric8CatalogWatchWithNamespacesIT.java | 258 ------------------ .../watcher-endpoint-slices-deployment.yaml | 33 --- .../app/watcher-endpoints-deployment.yaml | 4 +- .../integration/tests/commons/Commons.java | 10 +- .../tests/commons/fabric8_client/Util.java | 3 +- 8 files changed, 378 insertions(+), 339 deletions(-) create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchUtil.java create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchWithNamespacesDelegate.java delete mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchWithNamespacesIT.java delete mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/resources/app/watcher-endpoint-slices-deployment.yaml diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchIT.java index 8bd14f54..92360824 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchIT.java @@ -19,22 +19,17 @@ package org.springframework.cloud.kubernetes.fabric8.catalog.watch; import java.io.InputStream; import java.time.Duration; import java.util.List; -import java.util.Objects; +import java.util.Set; import io.fabric8.kubernetes.api.model.Service; import io.fabric8.kubernetes.api.model.apps.Deployment; import io.fabric8.kubernetes.api.model.networking.v1.Ingress; -import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.utils.Serialization; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.testcontainers.k3s.K3sContainer; -import reactor.netty.http.client.HttpClient; -import reactor.util.retry.Retry; -import reactor.util.retry.RetryBackoffSpec; import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; @@ -43,46 +38,61 @@ import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_cl import org.springframework.core.ParameterizedTypeReference; import org.springframework.core.ResolvableType; import org.springframework.http.HttpMethod; -import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.web.reactive.function.client.WebClient; import static org.awaitility.Awaitility.await; +import static org.springframework.cloud.kubernetes.fabric8.catalog.watch.Fabric8CatalogWatchUtil.builder; +import static org.springframework.cloud.kubernetes.fabric8.catalog.watch.Fabric8CatalogWatchUtil.patchForEndpointSlices; +import static org.springframework.cloud.kubernetes.fabric8.catalog.watch.Fabric8CatalogWatchUtil.patchForNamespaceFilterAndEndpointSlices; +import static org.springframework.cloud.kubernetes.fabric8.catalog.watch.Fabric8CatalogWatchUtil.patchForNamespaceFilterAndEndpoints; +import static org.springframework.cloud.kubernetes.fabric8.catalog.watch.Fabric8CatalogWatchUtil.retrySpec; +import static org.springframework.cloud.kubernetes.integration.tests.commons.Commons.pomVersion; /** * @author wind57 */ class Fabric8CatalogWatchIT { - private static final String APP_NAME = "spring-cloud-kubernetes-fabric8-client-catalog-watcher"; - private static final String NAMESPACE = "default"; - private static final K3sContainer K3S = Commons.container(); + public static final String NAMESPACE_A = "namespacea"; - private static KubernetesClient client; + public static final String NAMESPACE_B = "namespaceb"; + + private static final String IMAGE_NAME = "spring-cloud-kubernetes-fabric8-client-catalog-watcher"; + + private static final String DOCKER_IMAGE = "docker.io/springcloud/" + IMAGE_NAME + ":" + pomVersion(); + + private static final K3sContainer K3S = Commons.container(); private static Util util; @BeforeAll static void beforeAll() throws Exception { K3S.start(); - util = new Util(K3S); - client = util.client(); + Commons.validateImage(IMAGE_NAME, K3S); + Commons.loadSpringCloudKubernetesImage(IMAGE_NAME, K3S); - Commons.validateImage(APP_NAME, K3S); - Commons.loadSpringCloudKubernetesImage(APP_NAME, K3S); + util = new Util(K3S); + + util.createNamespace(NAMESPACE_A); + util.createNamespace(NAMESPACE_B); util.setUp(NAMESPACE); + util.setUpClusterWide(NAMESPACE, Set.of(NAMESPACE, NAMESPACE_A, NAMESPACE_B)); + util.busybox(NAMESPACE, Phase.CREATE); + + app(Phase.CREATE); } @AfterAll static void afterAll() { - Commons.systemPrune(); - } - @BeforeEach - void beforeEach() { - util.busybox(NAMESPACE, Phase.CREATE); + util.deleteNamespace(NAMESPACE_A); + util.deleteNamespace(NAMESPACE_B); + + app(Phase.DELETE); + Commons.systemPrune(); } /** @@ -95,18 +105,35 @@ class Fabric8CatalogWatchIT { */ @Test void testCatalogWatchWithEndpoints() throws Exception { - app(false, Phase.CREATE); - assertLogStatement("stateGenerator is of type: Fabric8EndpointsCatalogWatch"); + assertLogStatement(); test(); - app(false, Phase.DELETE); + + testCatalogWatchWithEndpointSlices(); + testCatalogWatchWithNamespaceFilterAndEndpoints(); + testCatalogWatchWithNamespaceFilterAndEndpointSlices(); } - @Test - void testCatalogWatchWithEndpointSlices() throws Exception { - app(true, Phase.CREATE); - assertLogStatement("stateGenerator is of type: Fabric8EndpointSliceV1CatalogWatch"); + void testCatalogWatchWithEndpointSlices() { + util.busybox(NAMESPACE, Phase.CREATE); + patchForEndpointSlices(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE); + Commons.waitForLogStatement("stateGenerator is of type: Fabric8EndpointSliceV1CatalogWatch", K3S, IMAGE_NAME); test(); - app(true, Phase.DELETE); + } + + void testCatalogWatchWithNamespaceFilterAndEndpoints() { + util.busybox(NAMESPACE_A, Phase.CREATE); + util.busybox(NAMESPACE_B, Phase.CREATE); + patchForNamespaceFilterAndEndpoints(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE); + Fabric8CatalogWatchWithNamespacesDelegate.testCatalogWatchWithNamespaceFilterAndEndpoints(K3S, IMAGE_NAME, + util); + } + + void testCatalogWatchWithNamespaceFilterAndEndpointSlices() { + util.busybox(NAMESPACE_A, Phase.CREATE); + util.busybox(NAMESPACE_B, Phase.CREATE); + patchForNamespaceFilterAndEndpointSlices(util, DOCKER_IMAGE, IMAGE_NAME, NAMESPACE); + Fabric8CatalogWatchWithNamespacesDelegate.testCatalogWatchWithNamespaceFilterAndEndpointSlices(K3S, IMAGE_NAME, + util); } /** @@ -114,13 +141,13 @@ class Fabric8CatalogWatchIT { * EndpointSlices. Here we make sure that in the test we actually use the correct * type. */ - private void assertLogStatement(String log) throws Exception { + private void assertLogStatement() throws Exception { String appPodName = K3S .execInContainer("kubectl", "get", "pods", "-l", "app=spring-cloud-kubernetes-fabric8-client-catalog-watcher", "-o=name", "--no-headers") .getStdout(); String allLogs = K3S.execInContainer("kubectl", "logs", appPodName.trim()).getStdout(); - Assertions.assertTrue(allLogs.contains(log)); + Assertions.assertTrue(allLogs.contains("stateGenerator is of type: Fabric8EndpointsCatalogWatch")); } /** @@ -143,6 +170,9 @@ class Fabric8CatalogWatchIT { // watcher implementation // we will get the first busybox instances here. if (result != null) { + if (result.size() != 3) { + return false; + } holder[0] = result.get(0); holder[1] = result.get(1); return true; @@ -190,21 +220,18 @@ class Fabric8CatalogWatchIT { return false; }); - Assertions.assertTrue(afterDelete[0].endpointName().contains(APP_NAME)); + Assertions.assertTrue(afterDelete[0].endpointName().contains(IMAGE_NAME)); Assertions.assertEquals("default", afterDelete[0].namespace()); } - private static void app(boolean useEndpointSlices, Phase phase) { + private static void app(Phase phase) { InputStream endpointsDeploymentStream = util.inputStream("app/watcher-endpoints-deployment.yaml"); - InputStream endpointSlicesDeploymentStream = util.inputStream("app/watcher-endpoint-slices-deployment.yaml"); InputStream serviceStream = util.inputStream("app/watcher-service.yaml"); InputStream ingressStream = util.inputStream("app/watcher-ingress.yaml"); - Deployment deployment = useEndpointSlices - ? Serialization.unmarshal(endpointSlicesDeploymentStream, Deployment.class) - : Serialization.unmarshal(endpointsDeploymentStream, Deployment.class); + Deployment deployment = Serialization.unmarshal(endpointsDeploymentStream, Deployment.class); Service service = Serialization.unmarshal(serviceStream, Service.class); Ingress ingress = Serialization.unmarshal(ingressStream, Ingress.class); @@ -217,12 +244,4 @@ class Fabric8CatalogWatchIT { } - private WebClient.Builder builder() { - return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create())); - } - - private RetryBackoffSpec retrySpec() { - return Retry.fixedDelay(15, Duration.ofSeconds(1)).filter(Objects::nonNull); - } - } diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchUtil.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchUtil.java new file mode 100644 index 00000000..16b9f818 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchUtil.java @@ -0,0 +1,156 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.fabric8.catalog.watch; + +import java.time.Duration; +import java.util.Map; +import java.util.Objects; + +import reactor.netty.http.client.HttpClient; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; +import org.springframework.web.reactive.function.client.WebClient; + +/** + * @author wind57 + */ +final class Fabric8CatalogWatchUtil { + + private static final Map POD_LABELS = Map.of("app", + "spring-cloud-kubernetes-fabric8-client-catalog-watcher"); + + private Fabric8CatalogWatchUtil() { + + } + + static final String BODY_ONE = """ + { + "spec": { + "template": { + "spec": { + "containers": [{ + "name": "spring-cloud-kubernetes-fabric8-client-catalog-watcher", + "image": "image_name_here", + "env": [ + { + "name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_FABRIC8_DISCOVERY", + "value": "DEBUG" + }, + { + "name": "SPRING_CLOUD_KUBERNETES_DISCOVERY_USE_ENDPOINT_SLICES", + "value": "TRUE" + } + ] + }] + } + } + } + } + """; + + static final String BODY_TWO = """ + { + "spec": { + "template": { + "spec": { + "containers": [{ + "name": "spring-cloud-kubernetes-fabric8-client-catalog-watcher", + "image": "image_name_here", + "env": [ + { + "name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_FABRIC8_DISCOVERY", + "value": "DEBUG" + }, + { + "name": "SPRING_CLOUD_KUBERNETES_DISCOVERY_USE_ENDPOINT_SLICES", + "value": "FALSE" + }, + { + "name": "SPRING_CLOUD_KUBERNETES_DISCOVERY_NAMESPACES_0", + "value": "namespacea" + }, + { + "name": "SPRING_CLOUD_KUBERNETES_DISCOVERY_NAMESPACES_1", + "value": "default" + } + ] + }] + } + } + } + } + """; + + static final String BODY_THREE = """ + { + "spec": { + "template": { + "spec": { + "containers": [{ + "name": "spring-cloud-kubernetes-fabric8-client-catalog-watcher", + "image": "image_name_here", + "env": [ + { + "name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_FABRIC8_DISCOVERY", + "value": "DEBUG" + }, + { + "name": "SPRING_CLOUD_KUBERNETES_DISCOVERY_USE_ENDPOINT_SLICES", + "value": "TRUE" + }, + { + "name": "SPRING_CLOUD_KUBERNETES_DISCOVERY_NAMESPACES_0", + "value": "namespacea" + }, + { + "name": "SPRING_CLOUD_KUBERNETES_DISCOVERY_NAMESPACES_1", + "value": "default" + } + ] + }] + } + } + } + } + """; + + static void patchForEndpointSlices(Util util, String dockerImage, String deploymentName, String namespace) { + util.patchWithReplace(dockerImage, deploymentName, namespace, BODY_ONE, POD_LABELS); + } + + static void patchForNamespaceFilterAndEndpoints(Util util, String dockerImage, String deploymentName, + String namespace) { + util.patchWithReplace(dockerImage, deploymentName, namespace, BODY_TWO, POD_LABELS); + } + + static void patchForNamespaceFilterAndEndpointSlices(Util util, String dockerImage, String deploymentName, + String namespace) { + util.patchWithReplace(dockerImage, deploymentName, namespace, BODY_THREE, POD_LABELS); + } + + static WebClient.Builder builder() { + return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create())); + } + + static RetryBackoffSpec retrySpec() { + return Retry.fixedDelay(15, Duration.ofSeconds(1)).filter(Objects::nonNull); + } + +} diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchWithNamespacesDelegate.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchWithNamespacesDelegate.java new file mode 100644 index 00000000..ad1c7543 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchWithNamespacesDelegate.java @@ -0,0 +1,146 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.fabric8.catalog.watch; + +import java.time.Duration; +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.testcontainers.k3s.K3sContainer; + +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; +import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; +import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.core.ResolvableType; +import org.springframework.http.HttpMethod; +import org.springframework.web.reactive.function.client.WebClient; + +import static org.awaitility.Awaitility.await; +import static org.springframework.cloud.kubernetes.fabric8.catalog.watch.Fabric8CatalogWatchIT.NAMESPACE_A; +import static org.springframework.cloud.kubernetes.fabric8.catalog.watch.Fabric8CatalogWatchIT.NAMESPACE_B; +import static org.springframework.cloud.kubernetes.fabric8.catalog.watch.Fabric8CatalogWatchUtil.builder; +import static org.springframework.cloud.kubernetes.fabric8.catalog.watch.Fabric8CatalogWatchUtil.retrySpec; + +/** + * @author wind57 + */ +final class Fabric8CatalogWatchWithNamespacesDelegate { + + private Fabric8CatalogWatchWithNamespacesDelegate() { + + } + + private static final String APP_NAME = "spring-cloud-kubernetes-fabric8-client-catalog-watcher"; + + /** + * 
+	 *     - we deploy one busybox service with 2 replica pods in namespace namespacea
+	 *     - we deploy one busybox service with 2 replica pods in namespace namespaceb
+	 *     - we enable the search to be made in namespacea and default ones
+	 *     - we receive an event from KubernetesCatalogWatcher, assert what is inside it
+	 *     - delete both busybox services in namespacea and namespaceb
+	 *     - assert that we receive only spring-cloud-kubernetes-fabric8-client-catalog-watcher pod
+	 *
+ */ + static void testCatalogWatchWithNamespaceFilterAndEndpoints(K3sContainer container, String imageName, Util util) { + Commons.waitForLogStatement("stateGenerator is of type: Fabric8EndpointsCatalogWatch", container, imageName); + test(util); + } + + static void testCatalogWatchWithNamespaceFilterAndEndpointSlices(K3sContainer container, String imageName, + Util util) { + Commons.waitForLogStatement("stateGenerator is of type: Fabric8EndpointSliceV1CatalogWatch", container, + imageName); + test(util); + } + + /** + * the test is the same for both endpoints and endpoint slices, the set-up for them is + * different. + */ + @SuppressWarnings("unchecked") + private static void test(Util util) { + + WebClient client = builder().baseUrl("http://localhost/result").build(); + EndpointNameAndNamespace[] holder = new EndpointNameAndNamespace[2]; + ResolvableType resolvableType = ResolvableType.forClassWithGenerics(List.class, EndpointNameAndNamespace.class); + + await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(240)).until(() -> { + List result = (List) client.method(HttpMethod.GET) + .retrieve().bodyToMono(ParameterizedTypeReference.forType(resolvableType.getType())) + .retryWhen(retrySpec()).block(); + + // we get 3 pods as input, but because they are sorted by name in the catalog + // watcher implementation + // we will get the first busybox instances here. + if (result != null) { + holder[0] = result.get(0); + holder[1] = result.get(1); + return true; + } + + return false; + }); + + EndpointNameAndNamespace resultOne = holder[0]; + EndpointNameAndNamespace resultTwo = holder[1]; + + Assertions.assertNotNull(resultOne); + Assertions.assertNotNull(resultTwo); + + Assertions.assertTrue(resultOne.endpointName().contains("busybox")); + Assertions.assertTrue(resultTwo.endpointName().contains("busybox")); + Assertions.assertEquals(NAMESPACE_A, resultOne.namespace()); + Assertions.assertEquals(NAMESPACE_A, resultTwo.namespace()); + + util.busybox(NAMESPACE_A, Phase.DELETE); + util.busybox(NAMESPACE_B, Phase.DELETE); + + // what we get after delete + EndpointNameAndNamespace[] afterDelete = new EndpointNameAndNamespace[1]; + + await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(240)).until(() -> { + List result = (List) client.method(HttpMethod.GET) + .retrieve().bodyToMono(ParameterizedTypeReference.forType(resolvableType.getType())) + .retryWhen(retrySpec()).block(); + + // we need to get the event from KubernetesCatalogWatch, but that happens + // on periodic bases. So in order to be sure we got the event we care about + // we wait until the result has a single entry, which means busybox was + // deleted + // + KubernetesCatalogWatch received the new update. + if (result != null && result.size() != 1) { + return false; + } + + // we will only receive one pod here, our own + if (result != null) { + afterDelete[0] = result.get(0); + return true; + } + + return false; + }); + + Assertions.assertTrue(afterDelete[0].endpointName().contains(APP_NAME)); + Assertions.assertEquals("default", afterDelete[0].namespace()); + + } + +} diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchWithNamespacesIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchWithNamespacesIT.java deleted file mode 100644 index bf998331..00000000 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchWithNamespacesIT.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Copyright 2013-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.kubernetes.fabric8.catalog.watch; - -import java.io.InputStream; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Set; - -import io.fabric8.kubernetes.api.model.EnvVar; -import io.fabric8.kubernetes.api.model.EnvVarBuilder; -import io.fabric8.kubernetes.api.model.Service; -import io.fabric8.kubernetes.api.model.apps.Deployment; -import io.fabric8.kubernetes.api.model.networking.v1.Ingress; -import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.utils.Serialization; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.testcontainers.k3s.K3sContainer; -import reactor.netty.http.client.HttpClient; -import reactor.util.retry.Retry; -import reactor.util.retry.RetryBackoffSpec; - -import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; -import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; -import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; -import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util; -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.core.ResolvableType; -import org.springframework.http.HttpMethod; -import org.springframework.http.client.reactive.ReactorClientHttpConnector; -import org.springframework.web.reactive.function.client.WebClient; - -import static org.awaitility.Awaitility.await; - -/** - * @author wind57 - */ -class Fabric8CatalogWatchWithNamespacesIT { - - private static final String APP_NAME = "spring-cloud-kubernetes-fabric8-client-catalog-watcher"; - - private static final String NAMESPACE_A = "namespacea"; - - private static final String NAMESPACE_B = "namespaceb"; - - private static final String NAMESPACE_DEFAULT = "default"; - - private static final K3sContainer K3S = Commons.container(); - - private static KubernetesClient client; - - private static Util util; - - @BeforeAll - static void beforeAll() throws Exception { - K3S.start(); - util = new Util(K3S); - client = util.client(); - - Commons.validateImage(APP_NAME, K3S); - Commons.loadSpringCloudKubernetesImage(APP_NAME, K3S); - - util.createNamespace(NAMESPACE_A); - util.createNamespace(NAMESPACE_B); - - util.setUpClusterWide(NAMESPACE_DEFAULT, Set.of(NAMESPACE_DEFAULT, NAMESPACE_A, NAMESPACE_B)); - } - - @BeforeEach - void beforeEach() { - util.busybox(NAMESPACE_A, Phase.CREATE); - util.busybox(NAMESPACE_B, Phase.CREATE); - } - - @AfterAll - static void afterAll() { - util.deleteNamespace(NAMESPACE_A); - util.deleteNamespace(NAMESPACE_B); - Commons.systemPrune(); - } - - /** - * 
-	 *     - we deploy one busybox service with 2 replica pods in namespace namespacea
-	 *     - we deploy one busybox service with 2 replica pods in namespace namespaceb
-	 *     - we enable the search to be made in namespacea and default ones
-	 *     - we receive an event from KubernetesCatalogWatcher, assert what is inside it
-	 *     - delete both busybox services in namespacea and namespaceb
-	 *     - assert that we receive only spring-cloud-kubernetes-fabric8-client-catalog-watcher pod
-	 *
- */ - @Test - void testCatalogWatchWithEndpoints() throws Exception { - app(false, Phase.CREATE); - assertLogStatement("stateGenerator is of type: Fabric8EndpointsCatalogWatch"); - test(); - app(false, Phase.DELETE); - } - - @Test - void testCatalogWatchWithEndpointSlices() throws Exception { - app(true, Phase.CREATE); - assertLogStatement("stateGenerator is of type: Fabric8EndpointSliceV1CatalogWatch"); - test(); - app(true, Phase.DELETE); - } - - /** - * we log in debug mode the type of the StateGenerator we use, be that Endpoints or - * EndpointSlices. Here we make sure that in the test we actually use the correct - * type. - */ - private void assertLogStatement(String log) throws Exception { - String appPodName = K3S - .execInContainer("kubectl", "get", "pods", "-l", - "app=spring-cloud-kubernetes-fabric8-client-catalog-watcher", "-o=name", "--no-headers") - .getStdout(); - String allLogs = K3S.execInContainer("kubectl", "logs", appPodName.trim()).getStdout(); - Assertions.assertTrue(allLogs.contains(log)); - } - - /** - * the test is the same for both endpoints and endpoint slices, the set-up for them is - * different. - */ - @SuppressWarnings("unchecked") - private void test() { - - WebClient client = builder().baseUrl("http://localhost/result").build(); - EndpointNameAndNamespace[] holder = new EndpointNameAndNamespace[2]; - ResolvableType resolvableType = ResolvableType.forClassWithGenerics(List.class, EndpointNameAndNamespace.class); - - await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(240)).until(() -> { - List result = (List) client.method(HttpMethod.GET) - .retrieve().bodyToMono(ParameterizedTypeReference.forType(resolvableType.getType())) - .retryWhen(retrySpec()).block(); - - // we get 3 pods as input, but because they are sorted by name in the catalog - // watcher implementation - // we will get the first busybox instances here. - if (result != null) { - holder[0] = result.get(0); - holder[1] = result.get(1); - return true; - } - - return false; - }); - - EndpointNameAndNamespace resultOne = holder[0]; - EndpointNameAndNamespace resultTwo = holder[1]; - - Assertions.assertNotNull(resultOne); - Assertions.assertNotNull(resultTwo); - - Assertions.assertTrue(resultOne.endpointName().contains("busybox")); - Assertions.assertTrue(resultTwo.endpointName().contains("busybox")); - Assertions.assertEquals(NAMESPACE_A, resultOne.namespace()); - Assertions.assertEquals(NAMESPACE_A, resultTwo.namespace()); - - util.busybox(NAMESPACE_A, Phase.DELETE); - util.busybox(NAMESPACE_B, Phase.DELETE); - - // what we get after delete - EndpointNameAndNamespace[] afterDelete = new EndpointNameAndNamespace[1]; - - await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(240)).until(() -> { - List result = (List) client.method(HttpMethod.GET) - .retrieve().bodyToMono(ParameterizedTypeReference.forType(resolvableType.getType())) - .retryWhen(retrySpec()).block(); - - // we need to get the event from KubernetesCatalogWatch, but that happens - // on periodic bases. So in order to be sure we got the event we care about - // we wait until the result has a single entry, which means busybox was - // deleted - // + KubernetesCatalogWatch received the new update. - if (result != null && result.size() != 1) { - return false; - } - - // we will only receive one pod here, our own - if (result != null) { - afterDelete[0] = result.get(0); - return true; - } - - return false; - }); - - Assertions.assertTrue(afterDelete[0].endpointName().contains(APP_NAME)); - Assertions.assertEquals("default", afterDelete[0].namespace()); - - } - - private static void app(boolean useEndpointSlices, Phase phase) { - - InputStream endpointsDeploymentStream = util.inputStream("app/watcher-endpoints-deployment.yaml"); - InputStream endpointSlicesDeploymentStream = util.inputStream("app/watcher-endpoint-slices-deployment.yaml"); - InputStream serviceStream = util.inputStream("app/watcher-service.yaml"); - InputStream ingressStream = util.inputStream("app/watcher-ingress.yaml"); - - Deployment deployment = useEndpointSlices - ? Serialization.unmarshal(endpointSlicesDeploymentStream, Deployment.class) - : Serialization.unmarshal(endpointsDeploymentStream, Deployment.class); - - List envVars = new ArrayList<>( - deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv()); - EnvVar namespaceAEnvVar = new EnvVarBuilder().withName("SPRING_CLOUD_KUBERNETES_DISCOVERY_NAMESPACES_0") - .withValue(NAMESPACE_A).build(); - EnvVar namespaceDefaultEnvVar = new EnvVarBuilder().withName("SPRING_CLOUD_KUBERNETES_DISCOVERY_NAMESPACES_1") - .withValue(NAMESPACE_DEFAULT).build(); - envVars.add(namespaceAEnvVar); - envVars.add(namespaceDefaultEnvVar); - - deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars); - - Service service = Serialization.unmarshal(serviceStream, Service.class); - Ingress ingress = Serialization.unmarshal(ingressStream, Ingress.class); - - if (phase.equals(Phase.CREATE)) { - util.createAndWait(Fabric8CatalogWatchWithNamespacesIT.NAMESPACE_DEFAULT, null, deployment, service, - ingress, true); - } - else { - util.deleteAndWait(Fabric8CatalogWatchWithNamespacesIT.NAMESPACE_DEFAULT, deployment, service, ingress); - } - - } - - private WebClient.Builder builder() { - return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create())); - } - - private RetryBackoffSpec retrySpec() { - return Retry.fixedDelay(15, Duration.ofSeconds(1)).filter(Objects::nonNull); - } - -} diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/resources/app/watcher-endpoint-slices-deployment.yaml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/resources/app/watcher-endpoint-slices-deployment.yaml deleted file mode 100644 index 531a1447..00000000 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/resources/app/watcher-endpoint-slices-deployment.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: spring-cloud-kubernetes-fabric8-client-catalog-watcher -spec: - selector: - matchLabels: - app: spring-cloud-kubernetes-fabric8-client-catalog-watcher - template: - metadata: - labels: - app: spring-cloud-kubernetes-fabric8-client-catalog-watcher - spec: - serviceAccountName: spring-cloud-kubernetes-serviceaccount - containers: - - name: spring-cloud-kubernetes-fabric8-client-catalog-watcher - image: docker.io/springcloud/spring-cloud-kubernetes-fabric8-client-catalog-watcher - imagePullPolicy: IfNotPresent - readinessProbe: - httpGet: - port: 8080 - path: /actuator/health/readiness - livenessProbe: - httpGet: - port: 8080 - path: /actuator/health/liveness - ports: - - containerPort: 8080 - env: - - name: LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_FABRIC8_DISCOVERY - value: DEBUG - - name: SPRING_CLOUD_KUBERNETES_DISCOVERY_USE_ENDPOINT_SLICES - value: true diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/resources/app/watcher-endpoints-deployment.yaml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/resources/app/watcher-endpoints-deployment.yaml index d2002401..fdce297c 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/resources/app/watcher-endpoints-deployment.yaml +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/resources/app/watcher-endpoints-deployment.yaml @@ -28,6 +28,6 @@ spec: - containerPort: 8080 env: - name: LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_FABRIC8_DISCOVERY - value: DEBUG + value: "DEBUG" - name: SPRING_CLOUD_KUBERNETES_DISCOVERY_USE_ENDPOINT_SLICES - value: false + value: "FALSE" diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java index f22caa9b..1c802a74 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/Commons.java @@ -158,7 +158,15 @@ public final class Commons { Files.copy(imageStream, imagePath); // import image with ctr. this works because TEMP_FOLDER is mounted in the // container - container.execInContainer("ctr", "i", "import", TEMP_FOLDER + "/" + tarName + ".tar"); + await().atMost(Duration.ofMinutes(2)).pollInterval(Duration.ofSeconds(1)).until(() -> { + Container.ExecResult result = container.execInContainer("ctr", "i", "import", + TEMP_FOLDER + "/" + tarName + ".tar"); + boolean noErrors = result.getStderr() == null || result.getStderr().isEmpty(); + if (!noErrors) { + LOG.info("error is : " + result.getStderr()); + } + return noErrors; + }); } } diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Util.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Util.java index a11f8b64..71d78601 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Util.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/fabric8_client/Util.java @@ -414,7 +414,8 @@ public final class Util { fail("No deployment with name " + deploymentName); } - Deployment deployment = deployments.getItems().get(0); + Deployment deployment = deployments.getItems().stream() + .filter(x -> x.getMetadata().getName().equals(deploymentName)).findFirst().orElseThrow(); // if no replicas are defined, it means only 1 is needed int replicas = Optional.ofNullable(deployment.getSpec().getReplicas()).orElse(1);