From 20e87ce16148f0150866485baefaea30d2953105 Mon Sep 17 00:00:00 2001 From: erabii Date: Wed, 4 Jan 2023 18:10:53 +0200 Subject: [PATCH] Fix 1042 : add k8s native catalog watch (#1177) --- .../catalog/KubernetesCatalogWatch.java | 126 +++++++ ...bernetesCatalogWatchAutoConfiguration.java | 55 +++ .../KubernetesCatalogWatchContext.java | 52 +++ .../KubernetesEndpointSlicesCatalogWatch.java | 104 ++++++ .../KubernetesEndpointsCatalogWatch.java | 113 ++++++ ...ot.autoconfigure.AutoConfiguration.imports | 1 + .../KubernetesCatalogWatchContextTests.java | 49 +++ ...rnetesCatalogWatchEndpointSlicesTests.java | 194 ++++++++++ .../KubernetesCatalogWatchEndpointsTests.java | 194 ++++++++++ ...atalogWatchEndpointSlicesSupportTests.java | 159 ++++++++ ...rnetesEndpointsAndEndpointSlicesTests.java | 193 ++++++++++ ...ConditionalOnKubernetesCatalogEnabled.java | 42 +++ .../KubernetesDiscoveryConstants.java | 15 + .../Fabric8EndpointSliceV1CatalogWatch.java | 6 +- .../discovery/KubernetesCatalogWatch.java | 8 +- ...bernetesCatalogWatchAutoConfiguration.java | 5 +- .../pom.xml | 1 + .../pom.xml | 116 ++++++ .../client/catalog/Application.java | 34 ++ .../EndpointNameAndNamespaceService.java | 43 +++ .../client/catalog/HeartBeatListener.java | 50 +++ .../client/catalog/HeartbeatController.java | 39 ++ .../KubernetesClientCatalogWatchIT.java | 304 +++++++++++++++ ...ernetesClientCatalogWatchNamespacesIT.java | 353 ++++++++++++++++++ .../watcher-endpoint-slices-deployment.yaml | 33 ++ .../app/watcher-endpoints-deployment.yaml | 33 ++ .../test/resources/app/watcher-ingress.yaml | 16 + .../test/resources/app/watcher-service.yaml | 14 + .../test/resources/busybox/deployment.yaml | 22 ++ .../src/test/resources/busybox/service.yaml | 12 + .../src/test/resources/logback-test.xml | 15 + ...atchIT.java => Fabric8CatalogWatchIT.java} | 2 +- ... Fabric8CatalogWatchWithNamespacesIT.java} | 2 +- .../integration/tests/commons/K8SUtils.java | 8 + 34 files changed, 2402 insertions(+), 11 deletions(-) create mode 100644 spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatch.java create mode 100644 spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchAutoConfiguration.java create mode 100644 spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchContext.java create mode 100644 spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesEndpointSlicesCatalogWatch.java create mode 100644 spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesEndpointsCatalogWatch.java create mode 100644 spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchContextTests.java create mode 100644 spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchEndpointSlicesTests.java create mode 100644 spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchEndpointsTests.java create mode 100644 spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesClientCatalogWatchEndpointSlicesSupportTests.java create mode 100644 spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesEndpointsAndEndpointSlicesTests.java create mode 100644 spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/ConditionalOnKubernetesCatalogEnabled.java create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/pom.xml create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/Application.java create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/EndpointNameAndNamespaceService.java create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/HeartBeatListener.java create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/HeartbeatController.java create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/client/catalog/KubernetesClientCatalogWatchIT.java create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/client/catalog/KubernetesClientCatalogWatchNamespacesIT.java create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-endpoint-slices-deployment.yaml create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-endpoints-deployment.yaml create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-ingress.yaml create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-service.yaml create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/busybox/deployment.yaml create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/busybox/service.yaml create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/logback-test.xml rename spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/{CatalogWatchIT.java => Fabric8CatalogWatchIT.java} (99%) rename spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/{CatalogWatchWithNamespacesIT.java => Fabric8CatalogWatchWithNamespacesIT.java} (99%) diff --git a/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatch.java b/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatch.java new file mode 100644 index 00000000..5d77f233 --- /dev/null +++ b/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatch.java @@ -0,0 +1,126 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.discovery.catalog; + +import java.util.List; +import java.util.function.Function; + +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.apis.CustomObjectsApi; +import io.kubernetes.client.openapi.models.V1APIResource; +import jakarta.annotation.PostConstruct; +import org.apache.commons.logging.LogFactory; + +import org.springframework.cloud.client.discovery.event.HeartbeatEvent; +import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider; +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.ApplicationEventPublisherAware; +import org.springframework.core.log.LogAccessor; +import org.springframework.scheduling.annotation.Scheduled; + +import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.DISCOVERY_GROUP; +import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.DISCOVERY_VERSION; +import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.ENDPOINT_SLICE; + +/** + * Catalog watch implementation for kubernetes native client. + * + * @author wind57 + */ +class KubernetesCatalogWatch implements ApplicationEventPublisherAware { + + private static final LogAccessor LOG = new LogAccessor(LogFactory.getLog(KubernetesCatalogWatch.class)); + + private final KubernetesCatalogWatchContext context; + + private Function> stateGenerator; + + private volatile List catalogEndpointsState = null; + + private ApplicationEventPublisher publisher; + + KubernetesCatalogWatch(CoreV1Api coreV1Api, ApiClient apiClient, KubernetesDiscoveryProperties properties, + KubernetesNamespaceProvider namespaceProvider) { + context = new KubernetesCatalogWatchContext(coreV1Api, apiClient, properties, namespaceProvider); + } + + @Override + public void setApplicationEventPublisher(ApplicationEventPublisher publisher) { + this.publisher = publisher; + } + + @Scheduled(fixedDelayString = "${spring.cloud.kubernetes.discovery.catalogServicesWatchDelay:30000}") + public void catalogServicesWatch() { + try { + + List currentState = stateGenerator.apply(context); + + if (!currentState.equals(catalogEndpointsState)) { + LOG.debug(() -> "Received endpoints update from kubernetesClient: " + currentState); + publisher.publishEvent(new HeartbeatEvent(this, currentState)); + } + + catalogEndpointsState = currentState; + } + catch (Exception e) { + LOG.error(e, () -> "Error watching Kubernetes Services"); + } + } + + @PostConstruct + void postConstruct() { + stateGenerator = stateGenerator(); + } + + Function> stateGenerator() { + + Function> localStateGenerator; + + if (context.properties().useEndpointSlices()) { + // this emulates : 'kubectl api-resources | grep -i EndpointSlice' + ApiClient apiClient = context.apiClient(); + CustomObjectsApi customObjectsApi = new CustomObjectsApi(apiClient); + try { + List resources = customObjectsApi.getAPIResources(DISCOVERY_GROUP, DISCOVERY_VERSION) + .getResources(); + boolean found = resources.stream().map(V1APIResource::getKind).anyMatch(ENDPOINT_SLICE::equals); + if (!found) { + throw new IllegalArgumentException("EndpointSlices are not supported on the cluster"); + } + else { + localStateGenerator = new KubernetesEndpointSlicesCatalogWatch(); + } + } + catch (ApiException e) { + throw new RuntimeException(e); + } + + } + else { + localStateGenerator = new KubernetesEndpointsCatalogWatch(); + } + + LOG.debug(() -> "stateGenerator is of type: " + localStateGenerator.getClass().getSimpleName()); + + return localStateGenerator; + } + +} diff --git a/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchAutoConfiguration.java b/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchAutoConfiguration.java new file mode 100644 index 00000000..a1c06068 --- /dev/null +++ b/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchAutoConfiguration.java @@ -0,0 +1,55 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.discovery.catalog; + +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.apis.CoreV1Api; + +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnCloudPlatform; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.cloud.CloudPlatform; +import org.springframework.cloud.client.ConditionalOnDiscoveryEnabled; +import org.springframework.cloud.kubernetes.client.KubernetesClientAutoConfiguration; +import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider; +import org.springframework.cloud.kubernetes.commons.discovery.ConditionalOnKubernetesCatalogEnabled; +import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; + +/** + * Auto configuration for catalog watcher. + * + * @author wind57 + */ +@Configuration(proxyBeanMethods = false) +@ConditionalOnDiscoveryEnabled +@ConditionalOnCloudPlatform(CloudPlatform.KUBERNETES) +@AutoConfigureAfter({ KubernetesClientAutoConfiguration.class }) +public class KubernetesCatalogWatchAutoConfiguration { + + @Bean + @ConditionalOnMissingBean + @ConditionalOnKubernetesCatalogEnabled + public KubernetesCatalogWatch kubernetesCatalogWatch(CoreV1Api coreV1Api, ApiClient apiClient, + KubernetesDiscoveryProperties properties, Environment environment) { + return new KubernetesCatalogWatch(coreV1Api, apiClient, properties, + new KubernetesNamespaceProvider(environment)); + } + +} diff --git a/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchContext.java b/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchContext.java new file mode 100644 index 00000000..42bf8915 --- /dev/null +++ b/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchContext.java @@ -0,0 +1,52 @@ +/* + * Copyright 2012-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.discovery.catalog; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1ObjectReference; + +import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider; +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties; + +/** + * A simple holder for some instances needed for either Endpoints or EndpointSlice catalog + * implementations. + * + * @author wind57 + */ +record KubernetesCatalogWatchContext(CoreV1Api coreV1Api, ApiClient apiClient, KubernetesDiscoveryProperties properties, + KubernetesNamespaceProvider namespaceProvider) { + + static List state(Stream references) { + return references.filter(Objects::nonNull).map(x -> new EndpointNameAndNamespace(x.getName(), x.getNamespace())) + .sorted(Comparator.comparing(EndpointNameAndNamespace::endpointName, String::compareTo)).toList(); + } + + static String labelSelector(Map labels) { + return labels.entrySet().stream().map(en -> en.getKey() + "=" + en.getValue()).collect(Collectors.joining("&")); + } + +} diff --git a/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesEndpointSlicesCatalogWatch.java b/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesEndpointSlicesCatalogWatch.java new file mode 100644 index 00000000..91a04797 --- /dev/null +++ b/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesEndpointSlicesCatalogWatch.java @@ -0,0 +1,104 @@ +/* + * Copyright 2012-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.discovery.catalog; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Stream; + +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.DiscoveryV1Api; +import io.kubernetes.client.openapi.models.V1Endpoint; +import io.kubernetes.client.openapi.models.V1EndpointSlice; +import io.kubernetes.client.openapi.models.V1ObjectReference; +import org.apache.commons.logging.LogFactory; + +import org.springframework.cloud.kubernetes.client.KubernetesClientUtils; +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.core.log.LogAccessor; + +import static org.springframework.cloud.kubernetes.client.discovery.catalog.KubernetesCatalogWatchContext.labelSelector; + +/** + * Implementation that is based on EndpointSlice V1. + * + * @author wind57 + */ +final class KubernetesEndpointSlicesCatalogWatch + implements Function> { + + private static final LogAccessor LOG = new LogAccessor( + LogFactory.getLog(KubernetesEndpointSlicesCatalogWatch.class)); + + @Override + public List apply(KubernetesCatalogWatchContext context) { + + List endpointSlices; + DiscoveryV1Api api = new DiscoveryV1Api(context.apiClient()); + + if (context.properties().allNamespaces()) { + LOG.debug(() -> "discovering endpoint slices in all namespaces"); + endpointSlices = endpointSlices(api, context.properties().serviceLabels()); + } + else if (!context.properties().namespaces().isEmpty()) { + LOG.debug(() -> "discovering endpoint slices in " + context.properties().namespaces()); + List inner = new ArrayList<>(context.properties().namespaces().size()); + context.properties().namespaces().forEach(namespace -> inner + .addAll(namespacedEndpointSlices(api, namespace, context.properties().serviceLabels()))); + endpointSlices = inner; + } + else { + String namespace = KubernetesClientUtils.getApplicationNamespace(null, "catalog-watch", + context.namespaceProvider()); + LOG.debug(() -> "discovering endpoint slices in namespace : " + namespace); + endpointSlices = namespacedEndpointSlices(api, namespace, context.properties().serviceLabels()); + } + + Stream references = endpointSlices.stream().map(V1EndpointSlice::getEndpoints) + .flatMap(List::stream).map(V1Endpoint::getTargetRef); + + return KubernetesCatalogWatchContext.state(references); + + } + + private List endpointSlices(DiscoveryV1Api api, Map labels) { + try { + return api.listEndpointSliceForAllNamespaces(null, null, null, labelSelector(labels), null, null, null, + null, null, null).getItems(); + } + catch (ApiException e) { + LOG.warn(e, () -> "can not list endpoint slices in all namespaces"); + return Collections.emptyList(); + } + } + + private List namespacedEndpointSlices(DiscoveryV1Api api, String namespace, + Map labels) { + try { + return api.listNamespacedEndpointSlice(namespace, null, null, null, null, labelSelector(labels), null, null, + null, null, null).getItems(); + } + catch (ApiException e) { + LOG.warn(e, () -> "can not list endpoint slices in namespace " + namespace); + return Collections.emptyList(); + } + } + +} diff --git a/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesEndpointsCatalogWatch.java b/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesEndpointsCatalogWatch.java new file mode 100644 index 00000000..2b11ba02 --- /dev/null +++ b/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesEndpointsCatalogWatch.java @@ -0,0 +1,113 @@ +/* + * Copyright 2012-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.discovery.catalog; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Stream; + +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1EndpointAddress; +import io.kubernetes.client.openapi.models.V1EndpointSubset; +import io.kubernetes.client.openapi.models.V1Endpoints; +import io.kubernetes.client.openapi.models.V1ObjectReference; +import org.apache.commons.logging.LogFactory; + +import org.springframework.cloud.kubernetes.client.KubernetesClientUtils; +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.core.log.LogAccessor; + +import static org.springframework.cloud.kubernetes.client.discovery.catalog.KubernetesCatalogWatchContext.labelSelector; + +/** + * Implementation that is based on V1Endpoints. + * + * @author wind57 + */ +final class KubernetesEndpointsCatalogWatch + implements Function> { + + private static final LogAccessor LOG = new LogAccessor(LogFactory.getLog(KubernetesEndpointsCatalogWatch.class)); + + @Override + public List apply(KubernetesCatalogWatchContext context) { + List endpoints; + CoreV1Api coreV1Api = context.coreV1Api(); + if (context.properties().allNamespaces()) { + LOG.debug(() -> "discovering endpoints in all namespaces"); + endpoints = endpoints(coreV1Api, context.properties().serviceLabels()); + } + else if (!context.properties().namespaces().isEmpty()) { + LOG.debug(() -> "discovering endpoints in " + context.properties().namespaces()); + List inner = new ArrayList<>(context.properties().namespaces().size()); + context.properties().namespaces().forEach(namespace -> inner + .addAll(namespacedEndpoints(coreV1Api, namespace, context.properties().serviceLabels()))); + endpoints = inner; + } + else { + String namespace = KubernetesClientUtils.getApplicationNamespace(null, "catalog-watch", + context.namespaceProvider()); + LOG.debug(() -> "discovering endpoints in namespace : " + namespace); + endpoints = namespacedEndpoints(coreV1Api, namespace, context.properties().serviceLabels()); + } + + /** + * 
+		 *   - An "V1Endpoints" holds a List of V1EndpointSubset.
+		 *   - A single V1EndpointSubset holds a List of V1EndpointAddress
+		 *
+		 *   - (The union of all V1EndpointSubsets is the Set of all V1Endpoints)
+		 *   - Set of V1Endpoints is the cartesian product of :
+		 *     V1EndpointSubset::getAddresses and V1EndpointSubset::getPorts (each is a List)
+		 *
+ */ + Stream references = endpoints.stream().map(V1Endpoints::getSubsets).filter(Objects::nonNull) + .flatMap(List::stream).map(V1EndpointSubset::getAddresses).filter(Objects::nonNull) + .flatMap(List::stream).map(V1EndpointAddress::getTargetRef); + + return KubernetesCatalogWatchContext.state(references); + + } + + private List endpoints(CoreV1Api client, Map labels) { + try { + return client.listEndpointsForAllNamespaces(null, null, null, labelSelector(labels), null, null, null, null, + null, null).getItems(); + } + catch (ApiException e) { + LOG.warn(e, () -> "can not list endpoints in all namespaces"); + return Collections.emptyList(); + } + } + + private List namespacedEndpoints(CoreV1Api client, String namespace, Map labels) { + try { + return client.listNamespacedEndpoints(namespace, null, null, null, null, labelSelector(labels), null, null, + null, null, null).getItems(); + } + catch (ApiException e) { + LOG.warn(e, () -> "can not list endpoints in namespace " + namespace); + return Collections.emptyList(); + } + } + +} diff --git a/spring-cloud-kubernetes-client-discovery/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/spring-cloud-kubernetes-client-discovery/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index 0b5d75f9..608352b9 100644 --- a/spring-cloud-kubernetes-client-discovery/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ b/spring-cloud-kubernetes-client-discovery/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -1,2 +1,3 @@ +org.springframework.cloud.kubernetes.client.discovery.catalog.KubernetesCatalogWatchAutoConfiguration org.springframework.cloud.kubernetes.client.discovery.KubernetesDiscoveryClientAutoConfiguration org.springframework.cloud.kubernetes.client.discovery.reactive.KubernetesInformerReactiveDiscoveryClientAutoConfiguration diff --git a/spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchContextTests.java b/spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchContextTests.java new file mode 100644 index 00000000..a52f0aea --- /dev/null +++ b/spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchContextTests.java @@ -0,0 +1,49 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.discovery.catalog; + +import java.util.Map; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * @author wind57 + */ +class KubernetesCatalogWatchContextTests { + + @Test + void emptyLabels() { + String result = KubernetesCatalogWatchContext.labelSelector(Map.of()); + Assertions.assertEquals("", result); + } + + @Test + void singleLabel() { + String result = KubernetesCatalogWatchContext.labelSelector(Map.of("a", "b")); + Assertions.assertEquals("a=b", result); + } + + @Test + void multipleLabelsLabel() { + String result = KubernetesCatalogWatchContext.labelSelector(Map.of("a", "b", "c", "d")); + Assertions.assertTrue(result.contains("c=d")); + Assertions.assertTrue(result.contains("&")); + Assertions.assertTrue(result.contains("a=b")); + } + +} diff --git a/spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchEndpointSlicesTests.java b/spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchEndpointSlicesTests.java new file mode 100644 index 00000000..6fb5b824 --- /dev/null +++ b/spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchEndpointSlicesTests.java @@ -0,0 +1,194 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.discovery.catalog; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.JSON; +import io.kubernetes.client.util.ClientBuilder; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; + +/** + * Test cases for the Endpoint Slices support + * + * @author wind57 + */ +class KubernetesCatalogWatchEndpointSlicesTests extends KubernetesEndpointsAndEndpointSlicesTests { + + private static final Boolean USE_ENDPOINT_SLICES = true; + + private static ApiClient apiClient; + + public static WireMockServer wireMockServer; + + @BeforeAll + static void beforeAll() { + wireMockServer = new WireMockServer(options().dynamicPort()); + wireMockServer.start(); + WireMock.configureFor(wireMockServer.port()); + apiClient = new ClientBuilder().setBasePath(wireMockServer.baseUrl()).build(); + } + + @AfterAll + public static void after() { + wireMockServer.stop(); + } + + @AfterEach + public void afterEach() { + WireMock.reset(); + Mockito.reset(APPLICATION_EVENT_PUBLISHER); + } + + @Test + @Override + void testInAllNamespacesEmptyServiceLabels() { + stubFor(get("/apis/discovery.k8s.io/v1/endpointslices?labelSelector=").willReturn( + aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("a", "default"))))); + KubernetesCatalogWatch watch = createWatcherInAllNamespacesWithLabels(Map.of(), Set.of(), null, apiClient, + USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "default"))); + } + + @Test + @Override + void testInAllNamespacesWithSingleLabel() { + stubFor(get("/apis/discovery.k8s.io/v1/endpointslices?labelSelector=a%3Db").willReturn( + aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("a", "default"))))); + KubernetesCatalogWatch watch = createWatcherInAllNamespacesWithLabels(Map.of("a", "b"), Set.of(), null, + apiClient, USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "default"))); + } + + @Test + @Override + void testInAllNamespacesWithDoubleLabel() { + stubFor(get("/apis/discovery.k8s.io/v1/endpointslices?labelSelector=a%3Db%26c%3Dd").willReturn( + aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("a", "default"))))); + // otherwise the stub might fail + LinkedHashMap map = new LinkedHashMap<>(); + map.put("a", "b"); + map.put("c", "d"); + KubernetesCatalogWatch watch = createWatcherInAllNamespacesWithLabels(map, Set.of(), null, apiClient, + USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "default"))); + } + + @Test + @Override + void testInSpecificNamespacesEmptyServiceLabels() { + stubFor(get("/apis/discovery.k8s.io/v1/namespaces/b/endpointslices?labelSelector=") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("a", "b"))))); + KubernetesCatalogWatch watch = createWatcherInSpecificNamespacesWithLabels(Set.of("b"), Map.of(), null, + apiClient, USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b"))); + } + + @Test + @Override + void testInSpecificNamespacesWithSingleLabel() { + stubFor(get("/apis/discovery.k8s.io/v1/namespaces/one/endpointslices?labelSelector=a%3Db") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("aa", "a"))))); + stubFor(get("/apis/discovery.k8s.io/v1/namespaces/two/endpointslices?labelSelector=a%3Db") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("bb", "b"))))); + + KubernetesCatalogWatch watch = createWatcherInSpecificNamespacesWithLabels(Set.of("one", "two"), + Map.of("a", "b"), null, apiClient, USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, + List.of(new EndpointNameAndNamespace("aa", "a"), new EndpointNameAndNamespace("bb", "b"))); + } + + @Test + @Override + void testInSpecificNamespacesWithDoubleLabel() { + stubFor(get("/apis/discovery.k8s.io/v1/namespaces/one/endpointslices?labelSelector=a%3Db%26c%3Dd") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("aa", "a"))))); + stubFor(get("/apis/discovery.k8s.io/v1/namespaces/two/endpointslices?labelSelector=a%3Db%26c%3Dd") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("bb", "b"))))); + + // otherwise the stub might fail + LinkedHashMap map = new LinkedHashMap<>(); + map.put("a", "b"); + map.put("c", "d"); + + KubernetesCatalogWatch watch = createWatcherInSpecificNamespacesWithLabels(Set.of("one", "two"), map, null, + apiClient, USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, + List.of(new EndpointNameAndNamespace("aa", "a"), new EndpointNameAndNamespace("bb", "b"))); + } + + @Test + @Override + void testInOneNamespaceEmptyServiceLabels() { + stubFor(get("/apis/discovery.k8s.io/v1/namespaces/b/endpointslices?labelSelector=") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("a", "b"))))); + KubernetesCatalogWatch watch = createWatcherInSpecificNamespaceWithLabels("b", Map.of(), null, apiClient, + USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b"))); + } + + @Test + @Override + void testInOneNamespaceWithSingleLabel() { + stubFor(get("/apis/discovery.k8s.io/v1/namespaces/b/endpointslices?labelSelector=key%3Dvalue") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("a", "b"))))); + KubernetesCatalogWatch watch = createWatcherInSpecificNamespaceWithLabels("b", Map.of("key", "value"), null, + apiClient, USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b"))); + } + + @Test + @Override + void testInOneNamespaceWithDoubleLabel() { + stubFor(get("/apis/discovery.k8s.io/v1/namespaces/b/endpointslices?labelSelector=key%3Dvalue%26key1%3Dvalue1") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("a", "b"))))); + // otherwise the stub might fail + LinkedHashMap map = new LinkedHashMap<>(); + map.put("key", "value"); + map.put("key1", "value1"); + KubernetesCatalogWatch watch = createWatcherInSpecificNamespaceWithLabels("b", map, null, apiClient, + USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b"))); + } + +} diff --git a/spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchEndpointsTests.java b/spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchEndpointsTests.java new file mode 100644 index 00000000..9ff7a1c4 --- /dev/null +++ b/spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatchEndpointsTests.java @@ -0,0 +1,194 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.discovery.catalog; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import io.kubernetes.client.openapi.JSON; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.util.ClientBuilder; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; + +/** + * Test cases for the Endpoints support + * + * @author wind57 + */ +class KubernetesCatalogWatchEndpointsTests extends KubernetesEndpointsAndEndpointSlicesTests { + + private static final Boolean USE_ENDPOINT_SLICES = false; + + private static CoreV1Api coreV1Api; + + public static WireMockServer wireMockServer; + + @BeforeAll + static void beforeAll() { + wireMockServer = new WireMockServer(options().dynamicPort()); + wireMockServer.start(); + WireMock.configureFor(wireMockServer.port()); + coreV1Api = new CoreV1Api(new ClientBuilder().setBasePath(wireMockServer.baseUrl()).build()); + } + + @AfterAll + public static void after() { + wireMockServer.stop(); + } + + @AfterEach + public void afterEach() { + WireMock.reset(); + Mockito.reset(APPLICATION_EVENT_PUBLISHER); + } + + @Test + @Override + void testInAllNamespacesEmptyServiceLabels() { + stubFor(get("/api/v1/endpoints?labelSelector=") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("a", "default"))))); + KubernetesCatalogWatch watch = createWatcherInAllNamespacesWithLabels(Map.of(), Set.of(), coreV1Api, null, + USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "default"))); + } + + @Test + @Override + void testInAllNamespacesWithSingleLabel() { + stubFor(get("/api/v1/endpoints?labelSelector=a%3Db") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("a", "default"))))); + KubernetesCatalogWatch watch = createWatcherInAllNamespacesWithLabels(Map.of("a", "b"), Set.of(), coreV1Api, + null, USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "default"))); + } + + @Test + @Override + void testInAllNamespacesWithDoubleLabel() { + stubFor(get("/api/v1/endpoints?labelSelector=a%3Db%26c%3Dd") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("a", "default"))))); + // otherwise the stub might fail + LinkedHashMap map = new LinkedHashMap<>(); + map.put("a", "b"); + map.put("c", "d"); + KubernetesCatalogWatch watch = createWatcherInAllNamespacesWithLabels(map, Set.of(), coreV1Api, null, + USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "default"))); + } + + @Test + @Override + void testInSpecificNamespacesEmptyServiceLabels() { + stubFor(get("/api/v1/namespaces/b/endpoints?labelSelector=") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("a", "b"))))); + KubernetesCatalogWatch watch = createWatcherInSpecificNamespacesWithLabels(Set.of("b"), Map.of(), coreV1Api, + null, USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b"))); + } + + @Test + @Override + void testInSpecificNamespacesWithSingleLabel() { + stubFor(get("/api/v1/namespaces/one/endpoints?labelSelector=a%3Db") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("aa", "a"))))); + stubFor(get("/api/v1/namespaces/two/endpoints?labelSelector=a%3Db") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("bb", "b"))))); + + KubernetesCatalogWatch watch = createWatcherInSpecificNamespacesWithLabels(Set.of("one", "two"), + Map.of("a", "b"), coreV1Api, null, USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, + List.of(new EndpointNameAndNamespace("aa", "a"), new EndpointNameAndNamespace("bb", "b"))); + } + + @Test + @Override + void testInSpecificNamespacesWithDoubleLabel() { + stubFor(get("/api/v1/namespaces/one/endpoints?labelSelector=a%3Db%26c%3Dd") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("aa", "a"))))); + stubFor(get("/api/v1/namespaces/two/endpoints?labelSelector=a%3Db%26c%3Dd") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("bb", "b"))))); + + // otherwise the stub might fail + LinkedHashMap map = new LinkedHashMap<>(); + map.put("a", "b"); + map.put("c", "d"); + + KubernetesCatalogWatch watch = createWatcherInSpecificNamespacesWithLabels(Set.of("one", "two"), map, coreV1Api, + null, USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, + List.of(new EndpointNameAndNamespace("aa", "a"), new EndpointNameAndNamespace("bb", "b"))); + } + + @Test + @Override + void testInOneNamespaceEmptyServiceLabels() { + stubFor(get("/api/v1/namespaces/b/endpoints?labelSelector=") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("a", "b"))))); + KubernetesCatalogWatch watch = createWatcherInSpecificNamespaceWithLabels("b", Map.of(), coreV1Api, null, + USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b"))); + } + + @Test + @Override + void testInOneNamespaceWithSingleLabel() { + stubFor(get("/api/v1/namespaces/b/endpoints?labelSelector=key%3Dvalue") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("a", "b"))))); + KubernetesCatalogWatch watch = createWatcherInSpecificNamespaceWithLabels("b", Map.of("key", "value"), + coreV1Api, null, USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b"))); + } + + @Test + @Override + void testInOneNamespaceWithDoubleLabel() { + stubFor(get("/api/v1/namespaces/b/endpoints?labelSelector=key%3Dvalue%26key1%3Dvalue1") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("a", "b"))))); + // otherwise the stub might fail + LinkedHashMap map = new LinkedHashMap<>(); + map.put("key", "value"); + map.put("key1", "value1"); + KubernetesCatalogWatch watch = createWatcherInSpecificNamespaceWithLabels("b", map, coreV1Api, null, + USE_ENDPOINT_SLICES); + + invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b"))); + } + +} diff --git a/spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesClientCatalogWatchEndpointSlicesSupportTests.java b/spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesClientCatalogWatchEndpointSlicesSupportTests.java new file mode 100644 index 00000000..efb5477d --- /dev/null +++ b/spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesClientCatalogWatchEndpointSlicesSupportTests.java @@ -0,0 +1,159 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.discovery.catalog; + +import java.util.Map; +import java.util.Set; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.JSON; +import io.kubernetes.client.openapi.models.V1APIResource; +import io.kubernetes.client.openapi.models.V1APIResourceBuilder; +import io.kubernetes.client.openapi.models.V1APIResourceList; +import io.kubernetes.client.openapi.models.V1APIResourceListBuilder; +import io.kubernetes.client.util.ClientBuilder; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider; +import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; +import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.ENDPOINT_SLICE; + +/** + * Tests that only assert the needed support for EndpointSlices in the cluster. + * + * @author wind57 + */ +class KubernetesClientCatalogWatchEndpointSlicesSupportTests { + + public static WireMockServer wireMockServer; + + private static final KubernetesNamespaceProvider NAMESPACE_PROVIDER = Mockito + .mock(KubernetesNamespaceProvider.class); + + private static ApiClient apiClient; + + @BeforeAll + static void beforeAll() { + wireMockServer = new WireMockServer(options().dynamicPort()); + wireMockServer.start(); + WireMock.configureFor(wireMockServer.port()); + + apiClient = new ClientBuilder().setBasePath(wireMockServer.baseUrl()).build(); + } + + @AfterAll + public static void after() { + wireMockServer.stop(); + } + + @AfterEach + public void afterEach() { + WireMock.reset(); + } + + /** + * 
+	 *     - endpoint slices are enabled, but are not supported by the cluster, as such we will fail
+	 *       with an IllegalArgumentException
+	 *     - V1APIResource is empty
+	 *
+ */ + @Test + void testEndpointSlicesEnabledButNotSupported() { + boolean useEndpointSlices = true; + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60, + false, "", Set.of(), Map.of(), "", null, 0, useEndpointSlices); + + V1APIResourceList list = new V1APIResourceListBuilder().addToResources(new V1APIResource()).build(); + stubFor(get("/apis/discovery.k8s.io/v1") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(list)))); + + KubernetesCatalogWatch watch = new KubernetesCatalogWatch(null, apiClient, properties, NAMESPACE_PROVIDER); + IllegalArgumentException ex = Assertions.assertThrows(IllegalArgumentException.class, watch::postConstruct); + Assertions.assertEquals("EndpointSlices are not supported on the cluster", ex.getMessage()); + } + + /** + * 
+	 *     - endpoint slices are enabled, but are not supported by the cluster, as such we will fail
+	 *       with an IllegalArgumentException
+	 *     - V1APIResource does not contain EndpointSlice
+	 *
+ */ + @Test + void testEndpointSlicesEnabledButNotSupportedViaApiVersions() { + boolean useEndpointSlices = true; + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60, + false, "", Set.of(), Map.of(), "", null, 0, useEndpointSlices); + + V1APIResourceList list = new V1APIResourceListBuilder() + .addToResources(new V1APIResourceBuilder().withName("not-the-one").build()).build(); + stubFor(get("/apis/discovery.k8s.io/v1") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(list)))); + + KubernetesCatalogWatch watch = new KubernetesCatalogWatch(null, apiClient, properties, NAMESPACE_PROVIDER); + IllegalArgumentException ex = Assertions.assertThrows(IllegalArgumentException.class, watch::postConstruct); + Assertions.assertEquals("EndpointSlices are not supported on the cluster", ex.getMessage()); + } + + /** + * endpoint slices are disabled via properties, as such we will use a catalog watch + * based on Endpoints + */ + @Test + void testEndpointsSupport() { + boolean useEndpointSlices = false; + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60, + false, "", Set.of(), Map.of(), "", null, 0, useEndpointSlices); + KubernetesCatalogWatch watch = new KubernetesCatalogWatch(null, apiClient, properties, NAMESPACE_PROVIDER); + + Assertions.assertEquals(KubernetesEndpointsCatalogWatch.class, watch.stateGenerator().getClass()); + } + + /** + * endpoint slices are enabled via properties and supported by the cluster, as such we + * will use a catalog watch based on Endpoint Slices + */ + @Test + void testEndpointSlicesSupport() { + boolean useEndpointSlices = true; + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60, + false, "", Set.of(), Map.of(), "", null, 0, useEndpointSlices); + + V1APIResourceList list = new V1APIResourceListBuilder() + .addToResources(new V1APIResourceBuilder().withName("endpointslices").withKind(ENDPOINT_SLICE).build()) + .build(); + stubFor(get("/apis/discovery.k8s.io/v1") + .willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(list)))); + + KubernetesCatalogWatch watch = new KubernetesCatalogWatch(null, apiClient, properties, NAMESPACE_PROVIDER); + Assertions.assertEquals(KubernetesEndpointSlicesCatalogWatch.class, watch.stateGenerator().getClass()); + } + +} diff --git a/spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesEndpointsAndEndpointSlicesTests.java b/spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesEndpointsAndEndpointSlicesTests.java new file mode 100644 index 00000000..389030f3 --- /dev/null +++ b/spring-cloud-kubernetes-client-discovery/src/test/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesEndpointsAndEndpointSlicesTests.java @@ -0,0 +1,193 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.discovery.catalog; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1EndpointAddressBuilder; +import io.kubernetes.client.openapi.models.V1EndpointBuilder; +import io.kubernetes.client.openapi.models.V1EndpointSliceBuilder; +import io.kubernetes.client.openapi.models.V1EndpointSliceList; +import io.kubernetes.client.openapi.models.V1EndpointSliceListBuilder; +import io.kubernetes.client.openapi.models.V1EndpointSubsetBuilder; +import io.kubernetes.client.openapi.models.V1EndpointsBuilder; +import io.kubernetes.client.openapi.models.V1EndpointsList; +import io.kubernetes.client.openapi.models.V1EndpointsListBuilder; +import io.kubernetes.client.openapi.models.V1ObjectReferenceBuilder; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import org.springframework.cloud.client.discovery.event.HeartbeatEvent; +import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider; +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties; +import org.springframework.context.ApplicationEventPublisher; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * make sure that all the tests for endpoints are also handled by endpoint slices + * + * @author wind57 + */ +abstract class KubernetesEndpointsAndEndpointSlicesTests { + + static final KubernetesNamespaceProvider NAMESPACE_PROVIDER = Mockito.mock(KubernetesNamespaceProvider.class); + + static final ArgumentCaptor HEARTBEAT_EVENT_ARGUMENT_CAPTOR = ArgumentCaptor + .forClass(HeartbeatEvent.class); + + static final ApplicationEventPublisher APPLICATION_EVENT_PUBLISHER = Mockito.mock(ApplicationEventPublisher.class); + + /** + * test in all namespaces with service labels being empty + */ + abstract void testInAllNamespacesEmptyServiceLabels(); + + /** + * test in all namespaces with service labels having a single label present + */ + abstract void testInAllNamespacesWithSingleLabel(); + + /** + * test in all namespaces with service labels having two labels + */ + abstract void testInAllNamespacesWithDoubleLabel(); + + /** + * test in some specific namespaces with service labels being empty + */ + abstract void testInSpecificNamespacesEmptyServiceLabels(); + + /** + * test in some specific namespaces with service labels having a single label present + */ + abstract void testInSpecificNamespacesWithSingleLabel(); + + /** + * test in some specific namespaces with service labels having two labels + */ + abstract void testInSpecificNamespacesWithDoubleLabel(); + + /** + * test in one namespace with service labels being empty + */ + abstract void testInOneNamespaceEmptyServiceLabels(); + + /** + * test in one namespace with service labels having a single label present + */ + abstract void testInOneNamespaceWithSingleLabel(); + + /** + * test in one namespace with service labels having two labels + */ + abstract void testInOneNamespaceWithDoubleLabel(); + + KubernetesCatalogWatch createWatcherInAllNamespacesWithLabels(Map labels, Set namespaces, + CoreV1Api coreV1Api, ApiClient apiClient, boolean endpointSlices) { + + boolean allNamespaces = true; + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, allNamespaces, namespaces, + true, 60, false, "", Set.of(), labels, "", null, 0, endpointSlices); + KubernetesCatalogWatch watch = new KubernetesCatalogWatch(coreV1Api, apiClient, properties, NAMESPACE_PROVIDER); + + if (endpointSlices) { + watch = Mockito.spy(watch); + Mockito.doReturn(new KubernetesEndpointSlicesCatalogWatch()).when(watch).stateGenerator(); + } + + watch.postConstruct(); + watch.setApplicationEventPublisher(APPLICATION_EVENT_PUBLISHER); + return watch; + + } + + KubernetesCatalogWatch createWatcherInSpecificNamespacesWithLabels(Set namespaces, + Map labels, CoreV1Api coreV1Api, ApiClient apiClient, boolean endpointSlices) { + + boolean allNamespaces = false; + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, allNamespaces, namespaces, + true, 60, false, "", Set.of(), labels, "", null, 0, false); + KubernetesCatalogWatch watch = new KubernetesCatalogWatch(coreV1Api, apiClient, properties, NAMESPACE_PROVIDER); + + if (endpointSlices) { + watch = Mockito.spy(watch); + Mockito.doReturn(new KubernetesEndpointSlicesCatalogWatch()).when(watch).stateGenerator(); + } + + watch.setApplicationEventPublisher(APPLICATION_EVENT_PUBLISHER); + watch.postConstruct(); + return watch; + + } + + KubernetesCatalogWatch createWatcherInSpecificNamespaceWithLabels(String namespace, Map labels, + CoreV1Api coreV1Api, ApiClient apiClient, boolean endpointSlices) { + + when(NAMESPACE_PROVIDER.getNamespace()).thenReturn(namespace); + + boolean allNamespaces = false; + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, allNamespaces, Set.of(), + true, 60, false, "", Set.of(), labels, "", null, 0, endpointSlices); + KubernetesCatalogWatch watch = new KubernetesCatalogWatch(coreV1Api, apiClient, properties, NAMESPACE_PROVIDER); + + if (endpointSlices) { + watch = Mockito.spy(watch); + Mockito.doReturn(new KubernetesEndpointSlicesCatalogWatch()).when(watch).stateGenerator(); + } + + watch.postConstruct(); + watch.setApplicationEventPublisher(APPLICATION_EVENT_PUBLISHER); + return watch; + + } + + V1EndpointsList endpoints(String name, String namespace) { + return new V1EndpointsListBuilder().addToItems(new V1EndpointsBuilder() + .addToSubsets(new V1EndpointSubsetBuilder().addToAddresses(new V1EndpointAddressBuilder() + .withTargetRef(new V1ObjectReferenceBuilder().withName(name).withNamespace(namespace).build()) + .build()).build()) + .build()).build(); + } + + V1EndpointSliceList endpointSlices(String name, String namespace) { + return new V1EndpointSliceListBuilder() + .addToItems(new V1EndpointSliceBuilder().addToEndpoints(new V1EndpointBuilder() + .withTargetRef(new V1ObjectReferenceBuilder().withName(name).withNamespace(namespace).build()) + .build()).build()) + .build(); + } + + static void invokeAndAssert(KubernetesCatalogWatch watch, List state) { + watch.catalogServicesWatch(); + + verify(APPLICATION_EVENT_PUBLISHER).publishEvent(HEARTBEAT_EVENT_ARGUMENT_CAPTOR.capture()); + + HeartbeatEvent event = HEARTBEAT_EVENT_ARGUMENT_CAPTOR.getValue(); + assertThat(event.getValue()).isInstanceOf(List.class); + + assertThat(event.getValue()).isEqualTo(state); + } + +} diff --git a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/ConditionalOnKubernetesCatalogEnabled.java b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/ConditionalOnKubernetesCatalogEnabled.java new file mode 100644 index 00000000..5d902bde --- /dev/null +++ b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/ConditionalOnKubernetesCatalogEnabled.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.commons.discovery; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +/** + * Provides a more succinct conditional + * spring.cloud.kubernetes.discovery.catalog-services-watch.enabled. + * + * @author wind57 + */ +@Target({ ElementType.TYPE, ElementType.METHOD }) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Inherited +@ConditionalOnProperty(value = "spring.cloud.kubernetes.discovery.catalog-services-watch.enabled", + matchIfMissing = true) +public @interface ConditionalOnKubernetesCatalogEnabled { + +} diff --git a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryConstants.java b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryConstants.java index 1cb23639..ba725854 100644 --- a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryConstants.java +++ b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryConstants.java @@ -52,4 +52,19 @@ public final class KubernetesDiscoveryConstants { */ public static final String UNSET_PORT_NAME = ""; + /** + * Discovery group for Catalog Watch. + */ + public static final String DISCOVERY_GROUP = "discovery.k8s.io"; + + /** + * Discovery version for Catalog Watch. + */ + public static final String DISCOVERY_VERSION = "v1"; + + /** + * Endpoint slice name. + */ + public static final String ENDPOINT_SLICE = "EndpointSlice"; + } diff --git a/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/Fabric8EndpointSliceV1CatalogWatch.java b/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/Fabric8EndpointSliceV1CatalogWatch.java index f121eebb..2e503ffe 100644 --- a/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/Fabric8EndpointSliceV1CatalogWatch.java +++ b/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/Fabric8EndpointSliceV1CatalogWatch.java @@ -48,12 +48,12 @@ final class Fabric8EndpointSliceV1CatalogWatch KubernetesClient client = context.kubernetesClient(); if (context.properties().allNamespaces()) { - LOG.debug(() -> "discovering endpoints in all namespaces"); + LOG.debug(() -> "discovering endpoint slices in all namespaces"); endpointSlices = client.discovery().v1().endpointSlices().inAnyNamespace() .withLabels(context.properties().serviceLabels()).list().getItems(); } else if (!context.properties().namespaces().isEmpty()) { - LOG.debug(() -> "discovering endpoints in " + context.properties().namespaces()); + LOG.debug(() -> "discovering endpoint slices in " + context.properties().namespaces()); List inner = new ArrayList<>(context.properties().namespaces().size()); context.properties().namespaces() .forEach(namespace -> inner.addAll(endpointSlices(context, namespace, client))); @@ -62,7 +62,7 @@ final class Fabric8EndpointSliceV1CatalogWatch else { String namespace = Fabric8Utils.getApplicationNamespace(context.kubernetesClient(), null, "catalog-watcher", context.namespaceProvider()); - LOG.debug(() -> "discovering endpoints in namespace : " + namespace); + LOG.debug(() -> "discovering endpoint slices in namespace : " + namespace); endpointSlices = endpointSlices(context, namespace, client); } diff --git a/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesCatalogWatch.java b/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesCatalogWatch.java index 69155bcf..7ab4e56d 100644 --- a/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesCatalogWatch.java +++ b/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesCatalogWatch.java @@ -35,14 +35,16 @@ import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.core.log.LogAccessor; import org.springframework.scheduling.annotation.Scheduled; +import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.DISCOVERY_GROUP; +import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.DISCOVERY_VERSION; +import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.ENDPOINT_SLICE; + /** * @author Oleg Vyukov */ public class KubernetesCatalogWatch implements ApplicationEventPublisherAware { - private static final String DISCOVERY_GROUP_VERSION = "discovery.k8s.io/v1"; - - private static final String ENDPOINT_SLICE = "EndpointSlice"; + private static final String DISCOVERY_GROUP_VERSION = DISCOVERY_GROUP + "/" + DISCOVERY_VERSION; private static final LogAccessor LOG = new LogAccessor(LogFactory.getLog(KubernetesCatalogWatch.class)); diff --git a/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesCatalogWatchAutoConfiguration.java b/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesCatalogWatchAutoConfiguration.java index c613b39c..d4f56566 100644 --- a/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesCatalogWatchAutoConfiguration.java +++ b/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesCatalogWatchAutoConfiguration.java @@ -21,10 +21,10 @@ import io.fabric8.kubernetes.client.KubernetesClient; import org.springframework.boot.autoconfigure.AutoConfigureAfter; import org.springframework.boot.autoconfigure.condition.ConditionalOnCloudPlatform; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.cloud.CloudPlatform; import org.springframework.cloud.client.ConditionalOnDiscoveryEnabled; import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider; +import org.springframework.cloud.kubernetes.commons.discovery.ConditionalOnKubernetesCatalogEnabled; import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties; import org.springframework.cloud.kubernetes.fabric8.Fabric8AutoConfiguration; import org.springframework.context.annotation.Bean; @@ -44,8 +44,7 @@ public class KubernetesCatalogWatchAutoConfiguration { @Bean @ConditionalOnMissingBean - @ConditionalOnProperty(name = "spring.cloud.kubernetes.discovery.catalog-services-watch.enabled", - matchIfMissing = true) + @ConditionalOnKubernetesCatalogEnabled public KubernetesCatalogWatch kubernetesCatalogWatch(KubernetesClient client, KubernetesDiscoveryProperties properties, Environment environment) { return new KubernetesCatalogWatch(client, properties, new KubernetesNamespaceProvider(environment)); diff --git a/spring-cloud-kubernetes-integration-tests/pom.xml b/spring-cloud-kubernetes-integration-tests/pom.xml index bece0b44..d050a1e4 100644 --- a/spring-cloud-kubernetes-integration-tests/pom.xml +++ b/spring-cloud-kubernetes-integration-tests/pom.xml @@ -91,5 +91,6 @@ spring-cloud-kubernetes-client-configmap-event-reload-multiple-apps spring-cloud-kubernetes-client-secrets-event-reload-multiple-apps spring-cloud-kubernetes-fabric8-client-catalog-watcher + spring-cloud-kubernetes-client-catalog-watcher diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/pom.xml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/pom.xml new file mode 100644 index 00000000..3c779fd5 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/pom.xml @@ -0,0 +1,116 @@ + + + + spring-cloud-kubernetes-integration-tests + org.springframework.cloud + 3.0.1-SNAPSHOT + + 4.0.0 + + spring-cloud-kubernetes-client-catalog-watcher + + + + org.springframework.cloud + spring-cloud-kubernetes-client-discovery + + + org.springframework.cloud + spring-cloud-kubernetes-test-support + + + org.springframework.boot + spring-boot-starter-webflux + + + org.springframework.boot + spring-boot-starter-actuator + + + com.github.docker-java + docker-java-core + test + + + com.github.docker-java + docker-java-transport-httpclient5 + test + + + + + + + ../src/main/resources + true + + + src/main/resources + true + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + docker.io/springcloud/${project.artifactId}:${project.version} + paketobuildpacks/builder + + + + build-image + + ${skip.build.image} + + package + + build-image + + + + repackage + package + + repackage + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + + integration-test + + + + + + ${testsToRun} + + + + + + + + diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/Application.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/Application.java new file mode 100644 index 00000000..d896ecb2 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/Application.java @@ -0,0 +1,34 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.catalog; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; + +/** + * @author wind57 + */ +@SpringBootApplication +@EnableScheduling +public class Application { + + public static void main(String[] args) { + SpringApplication.run(Application.class, args); + } + +} diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/EndpointNameAndNamespaceService.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/EndpointNameAndNamespaceService.java new file mode 100644 index 00000000..7f0c9cf7 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/EndpointNameAndNamespaceService.java @@ -0,0 +1,43 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.catalog; + +import java.util.List; + +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.stereotype.Service; + +/** + * holds an EndpointNameAndNamespace object, needed so that the controller can poll to see + * if anything has changed. And we call the controller from our tests. + * + * @author wind57 + */ +@Service +public class EndpointNameAndNamespaceService { + + private List result; + + public List result() { + return result; + } + + public void setResult(List result) { + this.result = result; + } + +} diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/HeartBeatListener.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/HeartBeatListener.java new file mode 100644 index 00000000..88ef4e73 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/HeartBeatListener.java @@ -0,0 +1,50 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.catalog; + +import java.util.List; + +import org.springframework.cloud.client.discovery.event.HeartbeatEvent; +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; + +/** + * Listener that will catch events from KubernetesCatalogWatch. + * + * @author wind57 + */ +@Component +public class HeartBeatListener implements ApplicationListener { + + private final EndpointNameAndNamespaceService service; + + public HeartBeatListener(EndpointNameAndNamespaceService service) { + this.service = service; + } + + @SuppressWarnings("unchecked") + @Override + public void onApplicationEvent(ApplicationEvent event) { + if (event instanceof HeartbeatEvent heartbeatEvent) { + List result = (List) heartbeatEvent.getValue(); + service.setResult(result); + } + } + +} diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/HeartbeatController.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/HeartbeatController.java new file mode 100644 index 00000000..6b6a1a01 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/main/java/org/springframework/cloud/kubernetes/client/catalog/HeartbeatController.java @@ -0,0 +1,39 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.catalog; + +import java.util.List; + +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class HeartbeatController { + + private final EndpointNameAndNamespaceService service; + + public HeartbeatController(EndpointNameAndNamespaceService service) { + this.service = service; + } + + @GetMapping("/result") + public List result() { + return service.result(); + } + +} diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/client/catalog/KubernetesClientCatalogWatchIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/client/catalog/KubernetesClientCatalogWatchIT.java new file mode 100644 index 00000000..07704366 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/client/catalog/KubernetesClientCatalogWatchIT.java @@ -0,0 +1,304 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.catalog; + +import java.time.Duration; +import java.util.List; +import java.util.Objects; + +import io.kubernetes.client.openapi.apis.AppsV1Api; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.apis.NetworkingV1Api; +import io.kubernetes.client.openapi.models.V1Deployment; +import io.kubernetes.client.openapi.models.V1Ingress; +import io.kubernetes.client.openapi.models.V1Service; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.k3s.K3sContainer; +import reactor.netty.http.client.HttpClient; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; +import org.springframework.cloud.kubernetes.integration.tests.commons.K8SUtils; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.core.ResolvableType; +import org.springframework.http.HttpMethod; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; +import org.springframework.web.reactive.function.client.WebClient; + +import static org.awaitility.Awaitility.await; +import static org.springframework.cloud.kubernetes.integration.tests.commons.K8SUtils.createApiClient; + +/** + * @author wind57 + */ +class KubernetesClientCatalogWatchIT { + + private static final String APP_NAME = "spring-cloud-kubernetes-client-catalog-watcher"; + + private static final String NAMESPACE = "default"; + + private static final K3sContainer K3S = Commons.container(); + + private static CoreV1Api api; + + private static AppsV1Api appsApi; + + private static NetworkingV1Api networkingApi; + + private static K8SUtils k8SUtils; + + private static String busyboxServiceName; + + private static String busyboxDeploymentName; + + private static String appDeploymentName; + + private static String appServiceName; + + private static String appIngressName; + + @BeforeAll + static void beforeAll() throws Exception { + K3S.start(); + + Commons.validateImage(APP_NAME, K3S); + Commons.loadSpringCloudKubernetesImage(APP_NAME, K3S); + + createApiClient(K3S.getKubeConfigYaml()); + api = new CoreV1Api(); + appsApi = new AppsV1Api(); + networkingApi = new NetworkingV1Api(); + k8SUtils = new K8SUtils(api, appsApi); + k8SUtils.setUp(NAMESPACE); + } + + @BeforeEach + void beforeEach() throws Exception { + deployBusyboxManifests(); + } + + @AfterEach + void afterEach() throws Exception { + deleteApp(); + } + + /** + * 
+	 *     - we deploy a busybox service with 2 replica pods
+	 *     - we receive an event from KubernetesCatalogWatcher, assert what is inside it
+	 *     - delete the busybox service
+	 *     - assert that we receive only spring-cloud-kubernetes-client-catalog-watcher pod
+	 *
+ */ + @Test + void testCatalogWatchWithEndpoints() throws Exception { + deployApp(false); + assertLogStatement("stateGenerator is of type: KubernetesEndpointsCatalogWatch"); + test(); + } + + @Test + void testCatalogWatchWithEndpointSlices() throws Exception { + deployApp(true); + assertLogStatement("stateGenerator is of type: KubernetesEndpointSlicesCatalogWatch"); + test(); + } + + /** + * we log in debug mode the type of the StateGenerator we use, be that Endpoints or + * EndpointSlices. Here we make sure that in the test we actually use the correct + * type. + */ + private void assertLogStatement(String log) throws Exception { + String appPodName = K3S.execInContainer("kubectl", "get", "pods", "-l", + "app=spring-cloud-kubernetes-client-catalog-watcher", "-o=name", "--no-headers").getStdout(); + String allLogs = K3S.execInContainer("kubectl", "logs", appPodName.trim()).getStdout(); + Assertions.assertTrue(allLogs.contains(log)); + } + + /** + * the test is the same for both endpoints and endpoint slices, the set-up for them is + * different. + */ + @SuppressWarnings("unchecked") + private void test() throws Exception { + + WebClient client = builder().baseUrl("localhost/result").build(); + EndpointNameAndNamespace[] holder = new EndpointNameAndNamespace[2]; + ResolvableType resolvableType = ResolvableType.forClassWithGenerics(List.class, EndpointNameAndNamespace.class); + + await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(240)).until(() -> { + List result = (List) client.method(HttpMethod.GET) + .retrieve().bodyToMono(ParameterizedTypeReference.forType(resolvableType.getType())) + .retryWhen(retrySpec()).block(); + + // we get 3 pods as input, but because they are sorted by name in the catalog + // watcher implementation + // we will get the first busybox instances here. + if (result != null) { + holder[0] = result.get(0); + holder[1] = result.get(1); + return true; + } + + return false; + }); + + EndpointNameAndNamespace resultOne = holder[0]; + EndpointNameAndNamespace resultTwo = holder[1]; + + Assertions.assertNotNull(resultOne); + Assertions.assertNotNull(resultTwo); + + Assertions.assertTrue(resultOne.endpointName().contains("busybox")); + Assertions.assertTrue(resultTwo.endpointName().contains("busybox")); + Assertions.assertEquals("default", resultOne.namespace()); + Assertions.assertEquals("default", resultTwo.namespace()); + + deleteBusyboxApp(); + + // what we get after delete + EndpointNameAndNamespace[] afterDelete = new EndpointNameAndNamespace[1]; + + await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(240)).until(() -> { + List result = (List) client.method(HttpMethod.GET) + .retrieve().bodyToMono(ParameterizedTypeReference.forType(resolvableType.getType())) + .retryWhen(retrySpec()).block(); + + // we need to get the event from KubernetesCatalogWatch, but that happens + // on periodic bases. So in order to be sure we got the event we care about + // we wait until the result has a single entry, which means busybox was + // deleted + // + KubernetesCatalogWatch received the new update. + if (result != null && result.size() != 1) { + return false; + } + + // we will only receive one pod here, our own + if (result != null) { + afterDelete[0] = result.get(0); + return true; + } + + return false; + }); + + Assertions.assertTrue(afterDelete[0].endpointName().contains(APP_NAME)); + Assertions.assertEquals("default", afterDelete[0].namespace()); + + } + + private void deployBusyboxManifests() throws Exception { + + V1Deployment busyboxDeployment = (V1Deployment) K8SUtils.readYamlFromClasspath(getBusyboxDeployment()); + + String[] image = K8SUtils.getImageFromDeployment(busyboxDeployment).split(":"); + Commons.pullImage(image[0], image[1], K3S); + Commons.loadImage(image[0], image[1], "busybox", K3S); + + appsApi.createNamespacedDeployment(NAMESPACE, busyboxDeployment, null, null, null, null); + busyboxDeploymentName = busyboxDeployment.getMetadata().getName(); + + V1Service busyboxService = (V1Service) K8SUtils.readYamlFromClasspath(getBusyboxService()); + busyboxServiceName = busyboxService.getMetadata().getName(); + api.createNamespacedService(NAMESPACE, busyboxService, null, null, null, null); + + k8SUtils.waitForDeployment(busyboxDeploymentName, NAMESPACE); + + } + + private static void deployApp(boolean useEndpointSlices) throws Exception { + + V1Deployment appDeployment = useEndpointSlices + ? (V1Deployment) K8SUtils.readYamlFromClasspath(getEndpointSlicesAppDeployment()) + : (V1Deployment) K8SUtils.readYamlFromClasspath(getEndpointsAppDeployment()); + + String version = K8SUtils.getPomVersion(); + String currentImage = appDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).getImage(); + appDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).setImage(currentImage + ":" + version); + + appsApi.createNamespacedDeployment(NAMESPACE, appDeployment, null, null, null, null); + appDeploymentName = appDeployment.getMetadata().getName(); + + V1Service appService = (V1Service) K8SUtils.readYamlFromClasspath(getAppService()); + appServiceName = appService.getMetadata().getName(); + api.createNamespacedService(NAMESPACE, appService, null, null, null, null); + + k8SUtils.waitForDeployment(appDeploymentName, NAMESPACE); + + V1Ingress appIngress = (V1Ingress) K8SUtils.readYamlFromClasspath(getAppIngress()); + appIngressName = appIngress.getMetadata().getName(); + networkingApi.createNamespacedIngress(NAMESPACE, appIngress, null, null, null, null); + k8SUtils.waitForIngress(appIngressName, NAMESPACE); + + } + + private void deleteBusyboxApp() throws Exception { + appsApi.deleteNamespacedDeployment(busyboxDeploymentName, NAMESPACE, null, null, null, null, null, null); + api.deleteNamespacedService(busyboxServiceName, NAMESPACE, null, null, null, null, null, null); + k8SUtils.waitForDeploymentToBeDeleted(busyboxDeploymentName, NAMESPACE); + } + + private void deleteApp() throws Exception { + appsApi.deleteNamespacedDeployment(appDeploymentName, NAMESPACE, null, null, null, null, null, null); + api.deleteNamespacedService(appServiceName, NAMESPACE, null, null, null, null, null, null); + networkingApi.deleteNamespacedIngress(appIngressName, NAMESPACE, null, null, null, null, null, null); + k8SUtils.waitForDeploymentToBeDeleted(busyboxDeploymentName, NAMESPACE); + } + + private static String getBusyboxService() { + return "busybox/service.yaml"; + } + + private static String getBusyboxDeployment() { + return "busybox/deployment.yaml"; + } + + /** + * deployment where support for endpoint slices is equal to false + */ + private static String getEndpointsAppDeployment() { + return "app/watcher-endpoints-deployment.yaml"; + } + + private static String getEndpointSlicesAppDeployment() { + return "app/watcher-endpoint-slices-deployment.yaml"; + } + + private static String getAppIngress() { + return "app/watcher-ingress.yaml"; + } + + private static String getAppService() { + return "app/watcher-service.yaml"; + } + + private WebClient.Builder builder() { + return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create())); + } + + private RetryBackoffSpec retrySpec() { + return Retry.fixedDelay(15, Duration.ofSeconds(1)).filter(Objects::nonNull); + } + +} diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/client/catalog/KubernetesClientCatalogWatchNamespacesIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/client/catalog/KubernetesClientCatalogWatchNamespacesIT.java new file mode 100644 index 00000000..458bb2a5 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/client/catalog/KubernetesClientCatalogWatchNamespacesIT.java @@ -0,0 +1,353 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.client.catalog; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +import io.kubernetes.client.openapi.apis.AppsV1Api; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.apis.NetworkingV1Api; +import io.kubernetes.client.openapi.models.V1Deployment; +import io.kubernetes.client.openapi.models.V1EnvVar; +import io.kubernetes.client.openapi.models.V1EnvVarBuilder; +import io.kubernetes.client.openapi.models.V1Ingress; +import io.kubernetes.client.openapi.models.V1NamespaceBuilder; +import io.kubernetes.client.openapi.models.V1Service; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.k3s.K3sContainer; +import reactor.netty.http.client.HttpClient; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; +import org.springframework.cloud.kubernetes.integration.tests.commons.K8SUtils; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.core.ResolvableType; +import org.springframework.http.HttpMethod; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; +import org.springframework.web.reactive.function.client.WebClient; + +import static org.awaitility.Awaitility.await; +import static org.springframework.cloud.kubernetes.integration.tests.commons.K8SUtils.createApiClient; + +public class KubernetesClientCatalogWatchNamespacesIT { + + private static final String APP_NAME = "spring-cloud-kubernetes-client-catalog-watcher"; + + private static final String NAMESPACE_A = "namespacea"; + + private static final String NAMESPACE_B = "namespaceb"; + + private static final String NAMESPACE_DEFAULT = "default"; + + private static final K3sContainer K3S = Commons.container(); + + private static CoreV1Api api; + + private static AppsV1Api appsApi; + + private static NetworkingV1Api networkingApi; + + private static K8SUtils k8SUtils; + + private static String busyboxServiceNameA; + + private static String busyboxServiceNameB; + + private static String busyboxDeploymentNameA; + + private static String busyboxDeploymentNameB; + + private static String appDeploymentName; + + private static String appServiceName; + + private static String appIngressName; + + @BeforeAll + static void beforeAll() throws Exception { + K3S.start(); + + Commons.validateImage(APP_NAME, K3S); + Commons.loadSpringCloudKubernetesImage(APP_NAME, K3S); + + createApiClient(K3S.getKubeConfigYaml()); + api = new CoreV1Api(); + appsApi = new AppsV1Api(); + networkingApi = new NetworkingV1Api(); + k8SUtils = new K8SUtils(api, appsApi); + k8SUtils.setUp(NAMESPACE_DEFAULT); + } + + @BeforeEach + void beforeEach() throws Exception { + api.createNamespace(new V1NamespaceBuilder().withNewMetadata().withName(NAMESPACE_A).and().build(), null, null, + null, null); + api.createNamespace(new V1NamespaceBuilder().withNewMetadata().withName(NAMESPACE_B).and().build(), null, null, + null, null); + k8SUtils.setUpClusterWide(NAMESPACE_DEFAULT, Set.of(NAMESPACE_A, NAMESPACE_B)); + deployBusyboxManifests(); + } + + @AfterEach + void afterEach() throws Exception { + deleteApp(); + k8SUtils.deleteNamespace(NAMESPACE_A); + k8SUtils.deleteNamespace(NAMESPACE_B); + } + + /** + * 
+	 *     - we deploy one busybox service with 2 replica pods in namespace namespacea
+	 *     - we deploy one busybox service with 2 replica pods in namespace namespaceb
+	 *     - we enable the search to be made in namespacea and default ones
+	 *     - we receive an event from KubernetesCatalogWatcher, assert what is inside it
+	 *     - delete both busybox services in namespacea and namespaceb
+	 *     - assert that we receive only spring-cloud-kubernetes-client-catalog-watcher pod
+	 *
+ */ + @Test + void testCatalogWatchWithEndpoints() throws Exception { + deployApp(false); + assertLogStatement("stateGenerator is of type: KubernetesEndpointsCatalogWatch"); + test(); + } + + @Test + void testCatalogWatchWithEndpointSlices() throws Exception { + deployApp(true); + assertLogStatement("stateGenerator is of type: KubernetesEndpointSlicesCatalogWatch"); + test(); + } + + /** + * we log in debug mode the type of the StateGenerator we use, be that Endpoints or + * EndpointSlices. Here we make sure that in the test we actually use the correct + * type. + */ + private void assertLogStatement(String log) throws Exception { + String appPodName = K3S + .execInContainer("kubectl", "get", "pods", "-l", + "app=spring-cloud-kubernetes-client-catalog-watcher", "-o=name", "--no-headers") + .getStdout(); + String allLogs = K3S.execInContainer("kubectl", "logs", appPodName.trim()).getStdout(); + Assertions.assertTrue(allLogs.contains(log)); + } + + /** + * the test is the same for both endpoints and endpoint slices, the set-up for them is + * different. + */ + @SuppressWarnings("unchecked") + private void test() throws Exception { + + WebClient client = builder().baseUrl("localhost/result").build(); + EndpointNameAndNamespace[] holder = new EndpointNameAndNamespace[2]; + ResolvableType resolvableType = ResolvableType.forClassWithGenerics(List.class, EndpointNameAndNamespace.class); + + await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(240)).until(() -> { + List result = (List) client.method(HttpMethod.GET) + .retrieve().bodyToMono(ParameterizedTypeReference.forType(resolvableType.getType())) + .retryWhen(retrySpec()).block(); + + // we get 3 pods as input, but because they are sorted by name in the catalog + // watcher implementation + // we will get the first busybox instances here. + if (result != null) { + holder[0] = result.get(0); + holder[1] = result.get(1); + return true; + } + + return false; + }); + + EndpointNameAndNamespace resultOne = holder[0]; + EndpointNameAndNamespace resultTwo = holder[1]; + + Assertions.assertNotNull(resultOne); + Assertions.assertNotNull(resultTwo); + + Assertions.assertTrue(resultOne.endpointName().contains("busybox")); + Assertions.assertTrue(resultTwo.endpointName().contains("busybox")); + Assertions.assertEquals(NAMESPACE_A, resultOne.namespace()); + Assertions.assertEquals(NAMESPACE_A, resultTwo.namespace()); + + deleteBusyboxApp(); + + // what we get after delete + EndpointNameAndNamespace[] afterDelete = new EndpointNameAndNamespace[1]; + + await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(240)).until(() -> { + List result = (List) client.method(HttpMethod.GET) + .retrieve().bodyToMono(ParameterizedTypeReference.forType(resolvableType.getType())) + .retryWhen(retrySpec()).block(); + + // we need to get the event from KubernetesCatalogWatch, but that happens + // on periodic bases. So in order to be sure we got the event we care about + // we wait until the result has a single entry, which means busybox was + // deleted + // + KubernetesCatalogWatch received the new update. + if (result != null && result.size() != 1) { + return false; + } + + // we will only receive one pod here, our own + if (result != null) { + afterDelete[0] = result.get(0); + return true; + } + + return false; + }); + + Assertions.assertTrue(afterDelete[0].endpointName().contains(APP_NAME)); + Assertions.assertEquals("default", afterDelete[0].namespace()); + + } + + private void deployBusyboxManifests() throws Exception { + + V1Deployment busyboxDeployment = (V1Deployment) K8SUtils.readYamlFromClasspath(getBusyboxDeployment()); + + String[] image = K8SUtils.getImageFromDeployment(busyboxDeployment).split(":"); + Commons.pullImage(image[0], image[1], K3S); + Commons.loadImage(image[0], image[1], "busybox", K3S); + + // namespace_a + appsApi.createNamespacedDeployment(NAMESPACE_A, busyboxDeployment, null, null, null, null); + busyboxDeploymentNameA = busyboxDeployment.getMetadata().getName(); + + V1Service busyboxServiceA = (V1Service) K8SUtils.readYamlFromClasspath(getBusyboxService()); + busyboxServiceNameA = busyboxServiceA.getMetadata().getName(); + api.createNamespacedService(NAMESPACE_A, busyboxServiceA, null, null, null, null); + + k8SUtils.waitForDeployment(busyboxDeploymentNameA, NAMESPACE_A); + + // namespace_b + appsApi.createNamespacedDeployment(NAMESPACE_B, busyboxDeployment, null, null, null, null); + busyboxDeploymentNameB = busyboxDeployment.getMetadata().getName(); + + V1Service busyboxServiceB = (V1Service) K8SUtils.readYamlFromClasspath(getBusyboxService()); + busyboxServiceNameB = busyboxServiceB.getMetadata().getName(); + api.createNamespacedService(NAMESPACE_B, busyboxServiceB, null, null, null, null); + + k8SUtils.waitForDeployment(busyboxDeploymentNameA, NAMESPACE_A); + + } + + private static void deployApp(boolean useEndpointSlices) throws Exception { + + V1Deployment appDeployment = useEndpointSlices + ? (V1Deployment) K8SUtils.readYamlFromClasspath(getEndpointSlicesAppDeployment()) + : (V1Deployment) K8SUtils.readYamlFromClasspath(getEndpointsAppDeployment()); + + String version = K8SUtils.getPomVersion(); + String currentImage = appDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).getImage(); + appDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).setImage(currentImage + ":" + version); + + List envVars = new ArrayList<>( + appDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv()); + V1EnvVar namespaceAEnvVar = new V1EnvVarBuilder().withName("SPRING_CLOUD_KUBERNETES_DISCOVERY_NAMESPACES_0") + .withValue(NAMESPACE_A).build(); + V1EnvVar namespaceDefaultEnvVar = new V1EnvVarBuilder() + .withName("SPRING_CLOUD_KUBERNETES_DISCOVERY_NAMESPACES_1").withValue(NAMESPACE_DEFAULT).build(); + envVars.add(namespaceAEnvVar); + envVars.add(namespaceDefaultEnvVar); + + appDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars); + + appsApi.createNamespacedDeployment(NAMESPACE_DEFAULT, appDeployment, null, null, null, null); + appDeploymentName = appDeployment.getMetadata().getName(); + + V1Service appService = (V1Service) K8SUtils.readYamlFromClasspath(getAppService()); + appServiceName = appService.getMetadata().getName(); + api.createNamespacedService(NAMESPACE_DEFAULT, appService, null, null, null, null); + + k8SUtils.waitForDeployment(appDeploymentName, NAMESPACE_DEFAULT); + + V1Ingress appIngress = (V1Ingress) K8SUtils.readYamlFromClasspath(getAppIngress()); + appIngressName = appIngress.getMetadata().getName(); + networkingApi.createNamespacedIngress(NAMESPACE_DEFAULT, appIngress, null, null, null, null); + + k8SUtils.waitForIngress(appIngressName, NAMESPACE_DEFAULT); + + } + + private void deleteBusyboxApp() throws Exception { + // namespacea + appsApi.deleteNamespacedDeployment(busyboxDeploymentNameA, NAMESPACE_A, null, null, null, null, null, null); + api.deleteNamespacedService(busyboxServiceNameA, NAMESPACE_A, null, null, null, null, null, null); + k8SUtils.waitForDeploymentToBeDeleted(busyboxDeploymentNameA, NAMESPACE_A); + + // namespaceb + appsApi.deleteNamespacedDeployment(busyboxDeploymentNameB, NAMESPACE_B, null, null, null, null, null, null); + api.deleteNamespacedService(busyboxServiceNameB, NAMESPACE_B, null, null, null, null, null, null); + k8SUtils.waitForDeploymentToBeDeleted(busyboxDeploymentNameB, NAMESPACE_B); + } + + private void deleteApp() throws Exception { + appsApi.deleteNamespacedDeployment(appDeploymentName, NAMESPACE_DEFAULT, null, null, null, null, null, null); + api.deleteNamespacedService(appServiceName, NAMESPACE_DEFAULT, null, null, null, null, null, null); + networkingApi.deleteNamespacedIngress(appIngressName, NAMESPACE_DEFAULT, null, null, null, null, null, null); + } + + private static String getBusyboxService() { + return "busybox/service.yaml"; + } + + private static String getBusyboxDeployment() { + return "busybox/deployment.yaml"; + } + + /** + * deployment where support for endpoint slices is equal to false + */ + private static String getEndpointsAppDeployment() { + return "app/watcher-endpoints-deployment.yaml"; + } + + private static String getEndpointSlicesAppDeployment() { + return "app/watcher-endpoint-slices-deployment.yaml"; + } + + private static String getAppIngress() { + return "app/watcher-ingress.yaml"; + } + + private static String getAppService() { + return "app/watcher-service.yaml"; + } + + private WebClient.Builder builder() { + return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create())); + } + + private RetryBackoffSpec retrySpec() { + return Retry.fixedDelay(15, Duration.ofSeconds(1)).filter(Objects::nonNull); + } + +} diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-endpoint-slices-deployment.yaml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-endpoint-slices-deployment.yaml new file mode 100644 index 00000000..4d97f2c5 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-endpoint-slices-deployment.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: spring-cloud-kubernetes-client-catalog-watcher +spec: + selector: + matchLabels: + app: spring-cloud-kubernetes-client-catalog-watcher + template: + metadata: + labels: + app: spring-cloud-kubernetes-client-catalog-watcher + spec: + serviceAccountName: spring-cloud-kubernetes-serviceaccount + containers: + - name: spring-cloud-kubernetes-client-catalog-watcher + image: docker.io/springcloud/spring-cloud-kubernetes-client-catalog-watcher + imagePullPolicy: IfNotPresent + readinessProbe: + httpGet: + port: 8080 + path: /actuator/health/readiness + livenessProbe: + httpGet: + port: 8080 + path: /actuator/health/liveness + ports: + - containerPort: 8080 + env: + - name: LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_CLIENT_DISCOVERY_CATALOG + value: DEBUG + - name: SPRING_CLOUD_KUBERNETES_DISCOVERY_USE_ENDPOINT_SLICES + value: true diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-endpoints-deployment.yaml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-endpoints-deployment.yaml new file mode 100644 index 00000000..c7b8a770 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-endpoints-deployment.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: spring-cloud-kubernetes-client-catalog-watcher +spec: + selector: + matchLabels: + app: spring-cloud-kubernetes-client-catalog-watcher + template: + metadata: + labels: + app: spring-cloud-kubernetes-client-catalog-watcher + spec: + serviceAccountName: spring-cloud-kubernetes-serviceaccount + containers: + - name: spring-cloud-kubernetes-client-catalog-watcher + image: docker.io/springcloud/spring-cloud-kubernetes-client-catalog-watcher + imagePullPolicy: IfNotPresent + readinessProbe: + httpGet: + port: 8080 + path: /actuator/health/readiness + livenessProbe: + httpGet: + port: 8080 + path: /actuator/health/liveness + ports: + - containerPort: 8080 + env: + - name: LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_CLIENT_DISCOVERY_CATALOG + value: DEBUG + - name: SPRING_CLOUD_KUBERNETES_DISCOVERY_USE_ENDPOINT_SLICES + value: false diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-ingress.yaml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-ingress.yaml new file mode 100644 index 00000000..cd01c1c2 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-ingress.yaml @@ -0,0 +1,16 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: spring-cloud-kubernetes-client-catalog-watcher-ingress + namespace: default +spec: + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: spring-cloud-kubernetes-client-catalog-watcher-service + port: + number: 8080 diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-service.yaml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-service.yaml new file mode 100644 index 00000000..53ea6aec --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/app/watcher-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: spring-cloud-kubernetes-client-catalog-watcher-service + name: spring-cloud-kubernetes-client-catalog-watcher-service +spec: + ports: + - name: http + port: 8080 + targetPort: 8080 + selector: + app: spring-cloud-kubernetes-client-catalog-watcher + type: ClusterIP diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/busybox/deployment.yaml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/busybox/deployment.yaml new file mode 100644 index 00000000..052a911e --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/busybox/deployment.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: busybox +spec: + selector: + matchLabels: + app: busybox + version: v1 + replicas: 2 + template: + metadata: + labels: + app: busybox + version: v1 + spec: + containers: + - name: busybox + # image: arm64/busybox:latest + image: busybox:1.35 + command: ["/bin/sh"] + args: ["-c", "sleep 100000"] diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/busybox/service.yaml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/busybox/service.yaml new file mode 100644 index 00000000..7a23a4d9 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/busybox/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: busybox-service +spec: + selector: + app: busybox + type: ClusterIP + ports: + - name: busybox-port + port: 8080 + targetPort: 80 diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/logback-test.xml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/logback-test.xml new file mode 100644 index 00000000..ee243343 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-client-catalog-watcher/src/test/resources/logback-test.xml @@ -0,0 +1,15 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n + + + + + + + + + + + diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/CatalogWatchIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchIT.java similarity index 99% rename from spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/CatalogWatchIT.java rename to spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchIT.java index 7e9c2daa..e35dc51a 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/CatalogWatchIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchIT.java @@ -52,7 +52,7 @@ import static org.awaitility.Awaitility.await; /** * @author wind57 */ -class CatalogWatchIT { +class Fabric8CatalogWatchIT { private static final String APP_NAME = "spring-cloud-kubernetes-fabric8-client-catalog-watcher"; diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/CatalogWatchWithNamespacesIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchWithNamespacesIT.java similarity index 99% rename from spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/CatalogWatchWithNamespacesIT.java rename to spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchWithNamespacesIT.java index 5f0f8166..a2a59fcc 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/CatalogWatchWithNamespacesIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-catalog-watcher/src/test/java/org/springframework/cloud/kubernetes/fabric8/catalog/watch/Fabric8CatalogWatchWithNamespacesIT.java @@ -57,7 +57,7 @@ import static org.awaitility.Awaitility.await; /** * @author wind57 */ -class CatalogWatchWithNamespacesIT { +class Fabric8CatalogWatchWithNamespacesIT { private static final String APP_NAME = "spring-cloud-kubernetes-fabric8-client-catalog-watcher"; diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/K8SUtils.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/K8SUtils.java index d81c4819..ad86be21 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/K8SUtils.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/integration/tests/commons/K8SUtils.java @@ -309,6 +309,14 @@ public class K8SUtils { () -> rbacApi.createNamespacedRole(namespace, role, null, null, null, null)); } + public void deleteNamespace(String name) throws Exception { + api.deleteNamespace(name, null, null, null, null, null, null); + + await().pollInterval(Duration.ofSeconds(1)).atMost(30, TimeUnit.SECONDS) + .until(() -> api.listNamespace(null, null, null, null, null, null, null, null, null, null).getItems() + .stream().noneMatch(x -> x.getMetadata().getName().equals(name))); + } + public void setUpClusterWide(String serviceAccountNamespace, Set namespaces) throws Exception { V1ServiceAccount serviceAccount = getConfigK8sClientItClusterServiceAccount();