Fix 1042 : add k8s native catalog watch (#1177)

This commit is contained in:
erabii
2023-01-04 18:10:53 +02:00
committed by GitHub
parent bc64eb8491
commit 20e87ce161
34 changed files with 2402 additions and 11 deletions

View File

@@ -0,0 +1,126 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.discovery.catalog;
import java.util.List;
import java.util.function.Function;
import io.kubernetes.client.openapi.ApiClient;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.apis.CustomObjectsApi;
import io.kubernetes.client.openapi.models.V1APIResource;
import jakarta.annotation.PostConstruct;
import org.apache.commons.logging.LogFactory;
import org.springframework.cloud.client.discovery.event.HeartbeatEvent;
import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider;
import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace;
import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.core.log.LogAccessor;
import org.springframework.scheduling.annotation.Scheduled;
import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.DISCOVERY_GROUP;
import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.DISCOVERY_VERSION;
import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.ENDPOINT_SLICE;
/**
* Catalog watch implementation for kubernetes native client.
*
* @author wind57
*/
class KubernetesCatalogWatch implements ApplicationEventPublisherAware {
private static final LogAccessor LOG = new LogAccessor(LogFactory.getLog(KubernetesCatalogWatch.class));
private final KubernetesCatalogWatchContext context;
private Function<KubernetesCatalogWatchContext, List<EndpointNameAndNamespace>> stateGenerator;
private volatile List<EndpointNameAndNamespace> catalogEndpointsState = null;
private ApplicationEventPublisher publisher;
KubernetesCatalogWatch(CoreV1Api coreV1Api, ApiClient apiClient, KubernetesDiscoveryProperties properties,
KubernetesNamespaceProvider namespaceProvider) {
context = new KubernetesCatalogWatchContext(coreV1Api, apiClient, properties, namespaceProvider);
}
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher publisher) {
this.publisher = publisher;
}
@Scheduled(fixedDelayString = "${spring.cloud.kubernetes.discovery.catalogServicesWatchDelay:30000}")
public void catalogServicesWatch() {
try {
List<EndpointNameAndNamespace> currentState = stateGenerator.apply(context);
if (!currentState.equals(catalogEndpointsState)) {
LOG.debug(() -> "Received endpoints update from kubernetesClient: " + currentState);
publisher.publishEvent(new HeartbeatEvent(this, currentState));
}
catalogEndpointsState = currentState;
}
catch (Exception e) {
LOG.error(e, () -> "Error watching Kubernetes Services");
}
}
@PostConstruct
void postConstruct() {
stateGenerator = stateGenerator();
}
Function<KubernetesCatalogWatchContext, List<EndpointNameAndNamespace>> stateGenerator() {
Function<KubernetesCatalogWatchContext, List<EndpointNameAndNamespace>> localStateGenerator;
if (context.properties().useEndpointSlices()) {
// this emulates : 'kubectl api-resources | grep -i EndpointSlice'
ApiClient apiClient = context.apiClient();
CustomObjectsApi customObjectsApi = new CustomObjectsApi(apiClient);
try {
List<V1APIResource> resources = customObjectsApi.getAPIResources(DISCOVERY_GROUP, DISCOVERY_VERSION)
.getResources();
boolean found = resources.stream().map(V1APIResource::getKind).anyMatch(ENDPOINT_SLICE::equals);
if (!found) {
throw new IllegalArgumentException("EndpointSlices are not supported on the cluster");
}
else {
localStateGenerator = new KubernetesEndpointSlicesCatalogWatch();
}
}
catch (ApiException e) {
throw new RuntimeException(e);
}
}
else {
localStateGenerator = new KubernetesEndpointsCatalogWatch();
}
LOG.debug(() -> "stateGenerator is of type: " + localStateGenerator.getClass().getSimpleName());
return localStateGenerator;
}
}

View File

@@ -0,0 +1,55 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.discovery.catalog;
import io.kubernetes.client.openapi.ApiClient;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnCloudPlatform;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.cloud.CloudPlatform;
import org.springframework.cloud.client.ConditionalOnDiscoveryEnabled;
import org.springframework.cloud.kubernetes.client.KubernetesClientAutoConfiguration;
import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider;
import org.springframework.cloud.kubernetes.commons.discovery.ConditionalOnKubernetesCatalogEnabled;
import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.env.Environment;
/**
* Auto configuration for catalog watcher.
*
* @author wind57
*/
@Configuration(proxyBeanMethods = false)
@ConditionalOnDiscoveryEnabled
@ConditionalOnCloudPlatform(CloudPlatform.KUBERNETES)
@AutoConfigureAfter({ KubernetesClientAutoConfiguration.class })
public class KubernetesCatalogWatchAutoConfiguration {
@Bean
@ConditionalOnMissingBean
@ConditionalOnKubernetesCatalogEnabled
public KubernetesCatalogWatch kubernetesCatalogWatch(CoreV1Api coreV1Api, ApiClient apiClient,
KubernetesDiscoveryProperties properties, Environment environment) {
return new KubernetesCatalogWatch(coreV1Api, apiClient, properties,
new KubernetesNamespaceProvider(environment));
}
}

View File

@@ -0,0 +1,52 @@
/*
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.discovery.catalog;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import io.kubernetes.client.openapi.ApiClient;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.models.V1ObjectReference;
import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider;
import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace;
import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties;
/**
* A simple holder for some instances needed for either Endpoints or EndpointSlice catalog
* implementations.
*
* @author wind57
*/
record KubernetesCatalogWatchContext(CoreV1Api coreV1Api, ApiClient apiClient, KubernetesDiscoveryProperties properties,
KubernetesNamespaceProvider namespaceProvider) {
static List<EndpointNameAndNamespace> state(Stream<V1ObjectReference> references) {
return references.filter(Objects::nonNull).map(x -> new EndpointNameAndNamespace(x.getName(), x.getNamespace()))
.sorted(Comparator.comparing(EndpointNameAndNamespace::endpointName, String::compareTo)).toList();
}
static String labelSelector(Map<String, String> labels) {
return labels.entrySet().stream().map(en -> en.getKey() + "=" + en.getValue()).collect(Collectors.joining("&"));
}
}

View File

@@ -0,0 +1,104 @@
/*
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.discovery.catalog;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Stream;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.apis.DiscoveryV1Api;
import io.kubernetes.client.openapi.models.V1Endpoint;
import io.kubernetes.client.openapi.models.V1EndpointSlice;
import io.kubernetes.client.openapi.models.V1ObjectReference;
import org.apache.commons.logging.LogFactory;
import org.springframework.cloud.kubernetes.client.KubernetesClientUtils;
import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace;
import org.springframework.core.log.LogAccessor;
import static org.springframework.cloud.kubernetes.client.discovery.catalog.KubernetesCatalogWatchContext.labelSelector;
/**
* Implementation that is based on EndpointSlice V1.
*
* @author wind57
*/
final class KubernetesEndpointSlicesCatalogWatch
implements Function<KubernetesCatalogWatchContext, List<EndpointNameAndNamespace>> {
private static final LogAccessor LOG = new LogAccessor(
LogFactory.getLog(KubernetesEndpointSlicesCatalogWatch.class));
@Override
public List<EndpointNameAndNamespace> apply(KubernetesCatalogWatchContext context) {
List<V1EndpointSlice> endpointSlices;
DiscoveryV1Api api = new DiscoveryV1Api(context.apiClient());
if (context.properties().allNamespaces()) {
LOG.debug(() -> "discovering endpoint slices in all namespaces");
endpointSlices = endpointSlices(api, context.properties().serviceLabels());
}
else if (!context.properties().namespaces().isEmpty()) {
LOG.debug(() -> "discovering endpoint slices in " + context.properties().namespaces());
List<V1EndpointSlice> inner = new ArrayList<>(context.properties().namespaces().size());
context.properties().namespaces().forEach(namespace -> inner
.addAll(namespacedEndpointSlices(api, namespace, context.properties().serviceLabels())));
endpointSlices = inner;
}
else {
String namespace = KubernetesClientUtils.getApplicationNamespace(null, "catalog-watch",
context.namespaceProvider());
LOG.debug(() -> "discovering endpoint slices in namespace : " + namespace);
endpointSlices = namespacedEndpointSlices(api, namespace, context.properties().serviceLabels());
}
Stream<V1ObjectReference> references = endpointSlices.stream().map(V1EndpointSlice::getEndpoints)
.flatMap(List::stream).map(V1Endpoint::getTargetRef);
return KubernetesCatalogWatchContext.state(references);
}
private List<V1EndpointSlice> endpointSlices(DiscoveryV1Api api, Map<String, String> labels) {
try {
return api.listEndpointSliceForAllNamespaces(null, null, null, labelSelector(labels), null, null, null,
null, null, null).getItems();
}
catch (ApiException e) {
LOG.warn(e, () -> "can not list endpoint slices in all namespaces");
return Collections.emptyList();
}
}
private List<V1EndpointSlice> namespacedEndpointSlices(DiscoveryV1Api api, String namespace,
Map<String, String> labels) {
try {
return api.listNamespacedEndpointSlice(namespace, null, null, null, null, labelSelector(labels), null, null,
null, null, null).getItems();
}
catch (ApiException e) {
LOG.warn(e, () -> "can not list endpoint slices in namespace " + namespace);
return Collections.emptyList();
}
}
}

View File

@@ -0,0 +1,113 @@
/*
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.discovery.catalog;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Stream;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.models.V1EndpointAddress;
import io.kubernetes.client.openapi.models.V1EndpointSubset;
import io.kubernetes.client.openapi.models.V1Endpoints;
import io.kubernetes.client.openapi.models.V1ObjectReference;
import org.apache.commons.logging.LogFactory;
import org.springframework.cloud.kubernetes.client.KubernetesClientUtils;
import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace;
import org.springframework.core.log.LogAccessor;
import static org.springframework.cloud.kubernetes.client.discovery.catalog.KubernetesCatalogWatchContext.labelSelector;
/**
* Implementation that is based on V1Endpoints.
*
* @author wind57
*/
final class KubernetesEndpointsCatalogWatch
implements Function<KubernetesCatalogWatchContext, List<EndpointNameAndNamespace>> {
private static final LogAccessor LOG = new LogAccessor(LogFactory.getLog(KubernetesEndpointsCatalogWatch.class));
@Override
public List<EndpointNameAndNamespace> apply(KubernetesCatalogWatchContext context) {
List<V1Endpoints> endpoints;
CoreV1Api coreV1Api = context.coreV1Api();
if (context.properties().allNamespaces()) {
LOG.debug(() -> "discovering endpoints in all namespaces");
endpoints = endpoints(coreV1Api, context.properties().serviceLabels());
}
else if (!context.properties().namespaces().isEmpty()) {
LOG.debug(() -> "discovering endpoints in " + context.properties().namespaces());
List<V1Endpoints> inner = new ArrayList<>(context.properties().namespaces().size());
context.properties().namespaces().forEach(namespace -> inner
.addAll(namespacedEndpoints(coreV1Api, namespace, context.properties().serviceLabels())));
endpoints = inner;
}
else {
String namespace = KubernetesClientUtils.getApplicationNamespace(null, "catalog-watch",
context.namespaceProvider());
LOG.debug(() -> "discovering endpoints in namespace : " + namespace);
endpoints = namespacedEndpoints(coreV1Api, namespace, context.properties().serviceLabels());
}
/**
* <pre>
* - An "V1Endpoints" holds a List of V1EndpointSubset.
* - A single V1EndpointSubset holds a List of V1EndpointAddress
*
* - (The union of all V1EndpointSubsets is the Set of all V1Endpoints)
* - Set of V1Endpoints is the cartesian product of :
* V1EndpointSubset::getAddresses and V1EndpointSubset::getPorts (each is a List)
* </pre>
*/
Stream<V1ObjectReference> references = endpoints.stream().map(V1Endpoints::getSubsets).filter(Objects::nonNull)
.flatMap(List::stream).map(V1EndpointSubset::getAddresses).filter(Objects::nonNull)
.flatMap(List::stream).map(V1EndpointAddress::getTargetRef);
return KubernetesCatalogWatchContext.state(references);
}
private List<V1Endpoints> endpoints(CoreV1Api client, Map<String, String> labels) {
try {
return client.listEndpointsForAllNamespaces(null, null, null, labelSelector(labels), null, null, null, null,
null, null).getItems();
}
catch (ApiException e) {
LOG.warn(e, () -> "can not list endpoints in all namespaces");
return Collections.emptyList();
}
}
private List<V1Endpoints> namespacedEndpoints(CoreV1Api client, String namespace, Map<String, String> labels) {
try {
return client.listNamespacedEndpoints(namespace, null, null, null, null, labelSelector(labels), null, null,
null, null, null).getItems();
}
catch (ApiException e) {
LOG.warn(e, () -> "can not list endpoints in namespace " + namespace);
return Collections.emptyList();
}
}
}

View File

@@ -1,2 +1,3 @@
org.springframework.cloud.kubernetes.client.discovery.catalog.KubernetesCatalogWatchAutoConfiguration
org.springframework.cloud.kubernetes.client.discovery.KubernetesDiscoveryClientAutoConfiguration
org.springframework.cloud.kubernetes.client.discovery.reactive.KubernetesInformerReactiveDiscoveryClientAutoConfiguration

View File

@@ -0,0 +1,49 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.discovery.catalog;
import java.util.Map;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/**
* @author wind57
*/
class KubernetesCatalogWatchContextTests {
@Test
void emptyLabels() {
String result = KubernetesCatalogWatchContext.labelSelector(Map.of());
Assertions.assertEquals("", result);
}
@Test
void singleLabel() {
String result = KubernetesCatalogWatchContext.labelSelector(Map.of("a", "b"));
Assertions.assertEquals("a=b", result);
}
@Test
void multipleLabelsLabel() {
String result = KubernetesCatalogWatchContext.labelSelector(Map.of("a", "b", "c", "d"));
Assertions.assertTrue(result.contains("c=d"));
Assertions.assertTrue(result.contains("&"));
Assertions.assertTrue(result.contains("a=b"));
}
}

View File

@@ -0,0 +1,194 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.discovery.catalog;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.github.tomakehurst.wiremock.WireMockServer;
import com.github.tomakehurst.wiremock.client.WireMock;
import io.kubernetes.client.openapi.ApiClient;
import io.kubernetes.client.openapi.JSON;
import io.kubernetes.client.util.ClientBuilder;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace;
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
import static com.github.tomakehurst.wiremock.client.WireMock.get;
import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options;
/**
* Test cases for the Endpoint Slices support
*
* @author wind57
*/
class KubernetesCatalogWatchEndpointSlicesTests extends KubernetesEndpointsAndEndpointSlicesTests {
private static final Boolean USE_ENDPOINT_SLICES = true;
private static ApiClient apiClient;
public static WireMockServer wireMockServer;
@BeforeAll
static void beforeAll() {
wireMockServer = new WireMockServer(options().dynamicPort());
wireMockServer.start();
WireMock.configureFor(wireMockServer.port());
apiClient = new ClientBuilder().setBasePath(wireMockServer.baseUrl()).build();
}
@AfterAll
public static void after() {
wireMockServer.stop();
}
@AfterEach
public void afterEach() {
WireMock.reset();
Mockito.reset(APPLICATION_EVENT_PUBLISHER);
}
@Test
@Override
void testInAllNamespacesEmptyServiceLabels() {
stubFor(get("/apis/discovery.k8s.io/v1/endpointslices?labelSelector=").willReturn(
aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("a", "default")))));
KubernetesCatalogWatch watch = createWatcherInAllNamespacesWithLabels(Map.of(), Set.of(), null, apiClient,
USE_ENDPOINT_SLICES);
invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "default")));
}
@Test
@Override
void testInAllNamespacesWithSingleLabel() {
stubFor(get("/apis/discovery.k8s.io/v1/endpointslices?labelSelector=a%3Db").willReturn(
aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("a", "default")))));
KubernetesCatalogWatch watch = createWatcherInAllNamespacesWithLabels(Map.of("a", "b"), Set.of(), null,
apiClient, USE_ENDPOINT_SLICES);
invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "default")));
}
@Test
@Override
void testInAllNamespacesWithDoubleLabel() {
stubFor(get("/apis/discovery.k8s.io/v1/endpointslices?labelSelector=a%3Db%26c%3Dd").willReturn(
aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("a", "default")))));
// otherwise the stub might fail
LinkedHashMap<String, String> map = new LinkedHashMap<>();
map.put("a", "b");
map.put("c", "d");
KubernetesCatalogWatch watch = createWatcherInAllNamespacesWithLabels(map, Set.of(), null, apiClient,
USE_ENDPOINT_SLICES);
invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "default")));
}
@Test
@Override
void testInSpecificNamespacesEmptyServiceLabels() {
stubFor(get("/apis/discovery.k8s.io/v1/namespaces/b/endpointslices?labelSelector=")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("a", "b")))));
KubernetesCatalogWatch watch = createWatcherInSpecificNamespacesWithLabels(Set.of("b"), Map.of(), null,
apiClient, USE_ENDPOINT_SLICES);
invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b")));
}
@Test
@Override
void testInSpecificNamespacesWithSingleLabel() {
stubFor(get("/apis/discovery.k8s.io/v1/namespaces/one/endpointslices?labelSelector=a%3Db")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("aa", "a")))));
stubFor(get("/apis/discovery.k8s.io/v1/namespaces/two/endpointslices?labelSelector=a%3Db")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("bb", "b")))));
KubernetesCatalogWatch watch = createWatcherInSpecificNamespacesWithLabels(Set.of("one", "two"),
Map.of("a", "b"), null, apiClient, USE_ENDPOINT_SLICES);
invokeAndAssert(watch,
List.of(new EndpointNameAndNamespace("aa", "a"), new EndpointNameAndNamespace("bb", "b")));
}
@Test
@Override
void testInSpecificNamespacesWithDoubleLabel() {
stubFor(get("/apis/discovery.k8s.io/v1/namespaces/one/endpointslices?labelSelector=a%3Db%26c%3Dd")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("aa", "a")))));
stubFor(get("/apis/discovery.k8s.io/v1/namespaces/two/endpointslices?labelSelector=a%3Db%26c%3Dd")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("bb", "b")))));
// otherwise the stub might fail
LinkedHashMap<String, String> map = new LinkedHashMap<>();
map.put("a", "b");
map.put("c", "d");
KubernetesCatalogWatch watch = createWatcherInSpecificNamespacesWithLabels(Set.of("one", "two"), map, null,
apiClient, USE_ENDPOINT_SLICES);
invokeAndAssert(watch,
List.of(new EndpointNameAndNamespace("aa", "a"), new EndpointNameAndNamespace("bb", "b")));
}
@Test
@Override
void testInOneNamespaceEmptyServiceLabels() {
stubFor(get("/apis/discovery.k8s.io/v1/namespaces/b/endpointslices?labelSelector=")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("a", "b")))));
KubernetesCatalogWatch watch = createWatcherInSpecificNamespaceWithLabels("b", Map.of(), null, apiClient,
USE_ENDPOINT_SLICES);
invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b")));
}
@Test
@Override
void testInOneNamespaceWithSingleLabel() {
stubFor(get("/apis/discovery.k8s.io/v1/namespaces/b/endpointslices?labelSelector=key%3Dvalue")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("a", "b")))));
KubernetesCatalogWatch watch = createWatcherInSpecificNamespaceWithLabels("b", Map.of("key", "value"), null,
apiClient, USE_ENDPOINT_SLICES);
invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b")));
}
@Test
@Override
void testInOneNamespaceWithDoubleLabel() {
stubFor(get("/apis/discovery.k8s.io/v1/namespaces/b/endpointslices?labelSelector=key%3Dvalue%26key1%3Dvalue1")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpointSlices("a", "b")))));
// otherwise the stub might fail
LinkedHashMap<String, String> map = new LinkedHashMap<>();
map.put("key", "value");
map.put("key1", "value1");
KubernetesCatalogWatch watch = createWatcherInSpecificNamespaceWithLabels("b", map, null, apiClient,
USE_ENDPOINT_SLICES);
invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b")));
}
}

View File

@@ -0,0 +1,194 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.discovery.catalog;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.github.tomakehurst.wiremock.WireMockServer;
import com.github.tomakehurst.wiremock.client.WireMock;
import io.kubernetes.client.openapi.JSON;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.util.ClientBuilder;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace;
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
import static com.github.tomakehurst.wiremock.client.WireMock.get;
import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options;
/**
* Test cases for the Endpoints support
*
* @author wind57
*/
class KubernetesCatalogWatchEndpointsTests extends KubernetesEndpointsAndEndpointSlicesTests {
private static final Boolean USE_ENDPOINT_SLICES = false;
private static CoreV1Api coreV1Api;
public static WireMockServer wireMockServer;
@BeforeAll
static void beforeAll() {
wireMockServer = new WireMockServer(options().dynamicPort());
wireMockServer.start();
WireMock.configureFor(wireMockServer.port());
coreV1Api = new CoreV1Api(new ClientBuilder().setBasePath(wireMockServer.baseUrl()).build());
}
@AfterAll
public static void after() {
wireMockServer.stop();
}
@AfterEach
public void afterEach() {
WireMock.reset();
Mockito.reset(APPLICATION_EVENT_PUBLISHER);
}
@Test
@Override
void testInAllNamespacesEmptyServiceLabels() {
stubFor(get("/api/v1/endpoints?labelSelector=")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("a", "default")))));
KubernetesCatalogWatch watch = createWatcherInAllNamespacesWithLabels(Map.of(), Set.of(), coreV1Api, null,
USE_ENDPOINT_SLICES);
invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "default")));
}
@Test
@Override
void testInAllNamespacesWithSingleLabel() {
stubFor(get("/api/v1/endpoints?labelSelector=a%3Db")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("a", "default")))));
KubernetesCatalogWatch watch = createWatcherInAllNamespacesWithLabels(Map.of("a", "b"), Set.of(), coreV1Api,
null, USE_ENDPOINT_SLICES);
invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "default")));
}
@Test
@Override
void testInAllNamespacesWithDoubleLabel() {
stubFor(get("/api/v1/endpoints?labelSelector=a%3Db%26c%3Dd")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("a", "default")))));
// otherwise the stub might fail
LinkedHashMap<String, String> map = new LinkedHashMap<>();
map.put("a", "b");
map.put("c", "d");
KubernetesCatalogWatch watch = createWatcherInAllNamespacesWithLabels(map, Set.of(), coreV1Api, null,
USE_ENDPOINT_SLICES);
invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "default")));
}
@Test
@Override
void testInSpecificNamespacesEmptyServiceLabels() {
stubFor(get("/api/v1/namespaces/b/endpoints?labelSelector=")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("a", "b")))));
KubernetesCatalogWatch watch = createWatcherInSpecificNamespacesWithLabels(Set.of("b"), Map.of(), coreV1Api,
null, USE_ENDPOINT_SLICES);
invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b")));
}
@Test
@Override
void testInSpecificNamespacesWithSingleLabel() {
stubFor(get("/api/v1/namespaces/one/endpoints?labelSelector=a%3Db")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("aa", "a")))));
stubFor(get("/api/v1/namespaces/two/endpoints?labelSelector=a%3Db")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("bb", "b")))));
KubernetesCatalogWatch watch = createWatcherInSpecificNamespacesWithLabels(Set.of("one", "two"),
Map.of("a", "b"), coreV1Api, null, USE_ENDPOINT_SLICES);
invokeAndAssert(watch,
List.of(new EndpointNameAndNamespace("aa", "a"), new EndpointNameAndNamespace("bb", "b")));
}
@Test
@Override
void testInSpecificNamespacesWithDoubleLabel() {
stubFor(get("/api/v1/namespaces/one/endpoints?labelSelector=a%3Db%26c%3Dd")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("aa", "a")))));
stubFor(get("/api/v1/namespaces/two/endpoints?labelSelector=a%3Db%26c%3Dd")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("bb", "b")))));
// otherwise the stub might fail
LinkedHashMap<String, String> map = new LinkedHashMap<>();
map.put("a", "b");
map.put("c", "d");
KubernetesCatalogWatch watch = createWatcherInSpecificNamespacesWithLabels(Set.of("one", "two"), map, coreV1Api,
null, USE_ENDPOINT_SLICES);
invokeAndAssert(watch,
List.of(new EndpointNameAndNamespace("aa", "a"), new EndpointNameAndNamespace("bb", "b")));
}
@Test
@Override
void testInOneNamespaceEmptyServiceLabels() {
stubFor(get("/api/v1/namespaces/b/endpoints?labelSelector=")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("a", "b")))));
KubernetesCatalogWatch watch = createWatcherInSpecificNamespaceWithLabels("b", Map.of(), coreV1Api, null,
USE_ENDPOINT_SLICES);
invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b")));
}
@Test
@Override
void testInOneNamespaceWithSingleLabel() {
stubFor(get("/api/v1/namespaces/b/endpoints?labelSelector=key%3Dvalue")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("a", "b")))));
KubernetesCatalogWatch watch = createWatcherInSpecificNamespaceWithLabels("b", Map.of("key", "value"),
coreV1Api, null, USE_ENDPOINT_SLICES);
invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b")));
}
@Test
@Override
void testInOneNamespaceWithDoubleLabel() {
stubFor(get("/api/v1/namespaces/b/endpoints?labelSelector=key%3Dvalue%26key1%3Dvalue1")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(endpoints("a", "b")))));
// otherwise the stub might fail
LinkedHashMap<String, String> map = new LinkedHashMap<>();
map.put("key", "value");
map.put("key1", "value1");
KubernetesCatalogWatch watch = createWatcherInSpecificNamespaceWithLabels("b", map, coreV1Api, null,
USE_ENDPOINT_SLICES);
invokeAndAssert(watch, List.of(new EndpointNameAndNamespace("a", "b")));
}
}

View File

@@ -0,0 +1,159 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.discovery.catalog;
import java.util.Map;
import java.util.Set;
import com.github.tomakehurst.wiremock.WireMockServer;
import com.github.tomakehurst.wiremock.client.WireMock;
import io.kubernetes.client.openapi.ApiClient;
import io.kubernetes.client.openapi.JSON;
import io.kubernetes.client.openapi.models.V1APIResource;
import io.kubernetes.client.openapi.models.V1APIResourceBuilder;
import io.kubernetes.client.openapi.models.V1APIResourceList;
import io.kubernetes.client.openapi.models.V1APIResourceListBuilder;
import io.kubernetes.client.util.ClientBuilder;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider;
import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties;
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
import static com.github.tomakehurst.wiremock.client.WireMock.get;
import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options;
import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.ENDPOINT_SLICE;
/**
* Tests that only assert the needed support for EndpointSlices in the cluster.
*
* @author wind57
*/
class KubernetesClientCatalogWatchEndpointSlicesSupportTests {
public static WireMockServer wireMockServer;
private static final KubernetesNamespaceProvider NAMESPACE_PROVIDER = Mockito
.mock(KubernetesNamespaceProvider.class);
private static ApiClient apiClient;
@BeforeAll
static void beforeAll() {
wireMockServer = new WireMockServer(options().dynamicPort());
wireMockServer.start();
WireMock.configureFor(wireMockServer.port());
apiClient = new ClientBuilder().setBasePath(wireMockServer.baseUrl()).build();
}
@AfterAll
public static void after() {
wireMockServer.stop();
}
@AfterEach
public void afterEach() {
WireMock.reset();
}
/**
* <pre>
* - endpoint slices are enabled, but are not supported by the cluster, as such we will fail
* with an IllegalArgumentException
* - V1APIResource is empty
* </pre>
*/
@Test
void testEndpointSlicesEnabledButNotSupported() {
boolean useEndpointSlices = true;
KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60,
false, "", Set.of(), Map.of(), "", null, 0, useEndpointSlices);
V1APIResourceList list = new V1APIResourceListBuilder().addToResources(new V1APIResource()).build();
stubFor(get("/apis/discovery.k8s.io/v1")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(list))));
KubernetesCatalogWatch watch = new KubernetesCatalogWatch(null, apiClient, properties, NAMESPACE_PROVIDER);
IllegalArgumentException ex = Assertions.assertThrows(IllegalArgumentException.class, watch::postConstruct);
Assertions.assertEquals("EndpointSlices are not supported on the cluster", ex.getMessage());
}
/**
* <pre>
* - endpoint slices are enabled, but are not supported by the cluster, as such we will fail
* with an IllegalArgumentException
* - V1APIResource does not contain EndpointSlice
* </pre>
*/
@Test
void testEndpointSlicesEnabledButNotSupportedViaApiVersions() {
boolean useEndpointSlices = true;
KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60,
false, "", Set.of(), Map.of(), "", null, 0, useEndpointSlices);
V1APIResourceList list = new V1APIResourceListBuilder()
.addToResources(new V1APIResourceBuilder().withName("not-the-one").build()).build();
stubFor(get("/apis/discovery.k8s.io/v1")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(list))));
KubernetesCatalogWatch watch = new KubernetesCatalogWatch(null, apiClient, properties, NAMESPACE_PROVIDER);
IllegalArgumentException ex = Assertions.assertThrows(IllegalArgumentException.class, watch::postConstruct);
Assertions.assertEquals("EndpointSlices are not supported on the cluster", ex.getMessage());
}
/**
* endpoint slices are disabled via properties, as such we will use a catalog watch
* based on Endpoints
*/
@Test
void testEndpointsSupport() {
boolean useEndpointSlices = false;
KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60,
false, "", Set.of(), Map.of(), "", null, 0, useEndpointSlices);
KubernetesCatalogWatch watch = new KubernetesCatalogWatch(null, apiClient, properties, NAMESPACE_PROVIDER);
Assertions.assertEquals(KubernetesEndpointsCatalogWatch.class, watch.stateGenerator().getClass());
}
/**
* endpoint slices are enabled via properties and supported by the cluster, as such we
* will use a catalog watch based on Endpoint Slices
*/
@Test
void testEndpointSlicesSupport() {
boolean useEndpointSlices = true;
KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60,
false, "", Set.of(), Map.of(), "", null, 0, useEndpointSlices);
V1APIResourceList list = new V1APIResourceListBuilder()
.addToResources(new V1APIResourceBuilder().withName("endpointslices").withKind(ENDPOINT_SLICE).build())
.build();
stubFor(get("/apis/discovery.k8s.io/v1")
.willReturn(aResponse().withStatus(200).withBody(new JSON().serialize(list))));
KubernetesCatalogWatch watch = new KubernetesCatalogWatch(null, apiClient, properties, NAMESPACE_PROVIDER);
Assertions.assertEquals(KubernetesEndpointSlicesCatalogWatch.class, watch.stateGenerator().getClass());
}
}

View File

@@ -0,0 +1,193 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.discovery.catalog;
import java.util.List;
import java.util.Map;
import java.util.Set;
import io.kubernetes.client.openapi.ApiClient;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.models.V1EndpointAddressBuilder;
import io.kubernetes.client.openapi.models.V1EndpointBuilder;
import io.kubernetes.client.openapi.models.V1EndpointSliceBuilder;
import io.kubernetes.client.openapi.models.V1EndpointSliceList;
import io.kubernetes.client.openapi.models.V1EndpointSliceListBuilder;
import io.kubernetes.client.openapi.models.V1EndpointSubsetBuilder;
import io.kubernetes.client.openapi.models.V1EndpointsBuilder;
import io.kubernetes.client.openapi.models.V1EndpointsList;
import io.kubernetes.client.openapi.models.V1EndpointsListBuilder;
import io.kubernetes.client.openapi.models.V1ObjectReferenceBuilder;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import org.springframework.cloud.client.discovery.event.HeartbeatEvent;
import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider;
import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace;
import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties;
import org.springframework.context.ApplicationEventPublisher;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* make sure that all the tests for endpoints are also handled by endpoint slices
*
* @author wind57
*/
abstract class KubernetesEndpointsAndEndpointSlicesTests {
static final KubernetesNamespaceProvider NAMESPACE_PROVIDER = Mockito.mock(KubernetesNamespaceProvider.class);
static final ArgumentCaptor<HeartbeatEvent> HEARTBEAT_EVENT_ARGUMENT_CAPTOR = ArgumentCaptor
.forClass(HeartbeatEvent.class);
static final ApplicationEventPublisher APPLICATION_EVENT_PUBLISHER = Mockito.mock(ApplicationEventPublisher.class);
/**
* test in all namespaces with service labels being empty
*/
abstract void testInAllNamespacesEmptyServiceLabels();
/**
* test in all namespaces with service labels having a single label present
*/
abstract void testInAllNamespacesWithSingleLabel();
/**
* test in all namespaces with service labels having two labels
*/
abstract void testInAllNamespacesWithDoubleLabel();
/**
* test in some specific namespaces with service labels being empty
*/
abstract void testInSpecificNamespacesEmptyServiceLabels();
/**
* test in some specific namespaces with service labels having a single label present
*/
abstract void testInSpecificNamespacesWithSingleLabel();
/**
* test in some specific namespaces with service labels having two labels
*/
abstract void testInSpecificNamespacesWithDoubleLabel();
/**
* test in one namespace with service labels being empty
*/
abstract void testInOneNamespaceEmptyServiceLabels();
/**
* test in one namespace with service labels having a single label present
*/
abstract void testInOneNamespaceWithSingleLabel();
/**
* test in one namespace with service labels having two labels
*/
abstract void testInOneNamespaceWithDoubleLabel();
KubernetesCatalogWatch createWatcherInAllNamespacesWithLabels(Map<String, String> labels, Set<String> namespaces,
CoreV1Api coreV1Api, ApiClient apiClient, boolean endpointSlices) {
boolean allNamespaces = true;
KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, allNamespaces, namespaces,
true, 60, false, "", Set.of(), labels, "", null, 0, endpointSlices);
KubernetesCatalogWatch watch = new KubernetesCatalogWatch(coreV1Api, apiClient, properties, NAMESPACE_PROVIDER);
if (endpointSlices) {
watch = Mockito.spy(watch);
Mockito.doReturn(new KubernetesEndpointSlicesCatalogWatch()).when(watch).stateGenerator();
}
watch.postConstruct();
watch.setApplicationEventPublisher(APPLICATION_EVENT_PUBLISHER);
return watch;
}
KubernetesCatalogWatch createWatcherInSpecificNamespacesWithLabels(Set<String> namespaces,
Map<String, String> labels, CoreV1Api coreV1Api, ApiClient apiClient, boolean endpointSlices) {
boolean allNamespaces = false;
KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, allNamespaces, namespaces,
true, 60, false, "", Set.of(), labels, "", null, 0, false);
KubernetesCatalogWatch watch = new KubernetesCatalogWatch(coreV1Api, apiClient, properties, NAMESPACE_PROVIDER);
if (endpointSlices) {
watch = Mockito.spy(watch);
Mockito.doReturn(new KubernetesEndpointSlicesCatalogWatch()).when(watch).stateGenerator();
}
watch.setApplicationEventPublisher(APPLICATION_EVENT_PUBLISHER);
watch.postConstruct();
return watch;
}
KubernetesCatalogWatch createWatcherInSpecificNamespaceWithLabels(String namespace, Map<String, String> labels,
CoreV1Api coreV1Api, ApiClient apiClient, boolean endpointSlices) {
when(NAMESPACE_PROVIDER.getNamespace()).thenReturn(namespace);
boolean allNamespaces = false;
KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, allNamespaces, Set.of(),
true, 60, false, "", Set.of(), labels, "", null, 0, endpointSlices);
KubernetesCatalogWatch watch = new KubernetesCatalogWatch(coreV1Api, apiClient, properties, NAMESPACE_PROVIDER);
if (endpointSlices) {
watch = Mockito.spy(watch);
Mockito.doReturn(new KubernetesEndpointSlicesCatalogWatch()).when(watch).stateGenerator();
}
watch.postConstruct();
watch.setApplicationEventPublisher(APPLICATION_EVENT_PUBLISHER);
return watch;
}
V1EndpointsList endpoints(String name, String namespace) {
return new V1EndpointsListBuilder().addToItems(new V1EndpointsBuilder()
.addToSubsets(new V1EndpointSubsetBuilder().addToAddresses(new V1EndpointAddressBuilder()
.withTargetRef(new V1ObjectReferenceBuilder().withName(name).withNamespace(namespace).build())
.build()).build())
.build()).build();
}
V1EndpointSliceList endpointSlices(String name, String namespace) {
return new V1EndpointSliceListBuilder()
.addToItems(new V1EndpointSliceBuilder().addToEndpoints(new V1EndpointBuilder()
.withTargetRef(new V1ObjectReferenceBuilder().withName(name).withNamespace(namespace).build())
.build()).build())
.build();
}
static void invokeAndAssert(KubernetesCatalogWatch watch, List<EndpointNameAndNamespace> state) {
watch.catalogServicesWatch();
verify(APPLICATION_EVENT_PUBLISHER).publishEvent(HEARTBEAT_EVENT_ARGUMENT_CAPTOR.capture());
HeartbeatEvent event = HEARTBEAT_EVENT_ARGUMENT_CAPTOR.getValue();
assertThat(event.getValue()).isInstanceOf(List.class);
assertThat(event.getValue()).isEqualTo(state);
}
}

View File

@@ -0,0 +1,42 @@
/*
* Copyright 2019-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.commons.discovery;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
/**
* Provides a more succinct conditional
* <code>spring.cloud.kubernetes.discovery.catalog-services-watch.enabled</code>.
*
* @author wind57
*/
@Target({ ElementType.TYPE, ElementType.METHOD })
@Retention(RetentionPolicy.RUNTIME)
@Documented
@Inherited
@ConditionalOnProperty(value = "spring.cloud.kubernetes.discovery.catalog-services-watch.enabled",
matchIfMissing = true)
public @interface ConditionalOnKubernetesCatalogEnabled {
}

View File

@@ -52,4 +52,19 @@ public final class KubernetesDiscoveryConstants {
*/
public static final String UNSET_PORT_NAME = "<unset>";
/**
* Discovery group for Catalog Watch.
*/
public static final String DISCOVERY_GROUP = "discovery.k8s.io";
/**
* Discovery version for Catalog Watch.
*/
public static final String DISCOVERY_VERSION = "v1";
/**
* Endpoint slice name.
*/
public static final String ENDPOINT_SLICE = "EndpointSlice";
}

View File

@@ -48,12 +48,12 @@ final class Fabric8EndpointSliceV1CatalogWatch
KubernetesClient client = context.kubernetesClient();
if (context.properties().allNamespaces()) {
LOG.debug(() -> "discovering endpoints in all namespaces");
LOG.debug(() -> "discovering endpoint slices in all namespaces");
endpointSlices = client.discovery().v1().endpointSlices().inAnyNamespace()
.withLabels(context.properties().serviceLabels()).list().getItems();
}
else if (!context.properties().namespaces().isEmpty()) {
LOG.debug(() -> "discovering endpoints in " + context.properties().namespaces());
LOG.debug(() -> "discovering endpoint slices in " + context.properties().namespaces());
List<EndpointSlice> inner = new ArrayList<>(context.properties().namespaces().size());
context.properties().namespaces()
.forEach(namespace -> inner.addAll(endpointSlices(context, namespace, client)));
@@ -62,7 +62,7 @@ final class Fabric8EndpointSliceV1CatalogWatch
else {
String namespace = Fabric8Utils.getApplicationNamespace(context.kubernetesClient(), null, "catalog-watcher",
context.namespaceProvider());
LOG.debug(() -> "discovering endpoints in namespace : " + namespace);
LOG.debug(() -> "discovering endpoint slices in namespace : " + namespace);
endpointSlices = endpointSlices(context, namespace, client);
}

View File

@@ -35,14 +35,16 @@ import org.springframework.context.ApplicationEventPublisherAware;
import org.springframework.core.log.LogAccessor;
import org.springframework.scheduling.annotation.Scheduled;
import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.DISCOVERY_GROUP;
import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.DISCOVERY_VERSION;
import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.ENDPOINT_SLICE;
/**
* @author Oleg Vyukov
*/
public class KubernetesCatalogWatch implements ApplicationEventPublisherAware {
private static final String DISCOVERY_GROUP_VERSION = "discovery.k8s.io/v1";
private static final String ENDPOINT_SLICE = "EndpointSlice";
private static final String DISCOVERY_GROUP_VERSION = DISCOVERY_GROUP + "/" + DISCOVERY_VERSION;
private static final LogAccessor LOG = new LogAccessor(LogFactory.getLog(KubernetesCatalogWatch.class));

View File

@@ -21,10 +21,10 @@ import io.fabric8.kubernetes.client.KubernetesClient;
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnCloudPlatform;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.cloud.CloudPlatform;
import org.springframework.cloud.client.ConditionalOnDiscoveryEnabled;
import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider;
import org.springframework.cloud.kubernetes.commons.discovery.ConditionalOnKubernetesCatalogEnabled;
import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties;
import org.springframework.cloud.kubernetes.fabric8.Fabric8AutoConfiguration;
import org.springframework.context.annotation.Bean;
@@ -44,8 +44,7 @@ public class KubernetesCatalogWatchAutoConfiguration {
@Bean
@ConditionalOnMissingBean
@ConditionalOnProperty(name = "spring.cloud.kubernetes.discovery.catalog-services-watch.enabled",
matchIfMissing = true)
@ConditionalOnKubernetesCatalogEnabled
public KubernetesCatalogWatch kubernetesCatalogWatch(KubernetesClient client,
KubernetesDiscoveryProperties properties, Environment environment) {
return new KubernetesCatalogWatch(client, properties, new KubernetesNamespaceProvider(environment));

View File

@@ -91,5 +91,6 @@
<module>spring-cloud-kubernetes-client-configmap-event-reload-multiple-apps</module>
<module>spring-cloud-kubernetes-client-secrets-event-reload-multiple-apps</module>
<module>spring-cloud-kubernetes-fabric8-client-catalog-watcher</module>
<module>spring-cloud-kubernetes-client-catalog-watcher</module>
</modules>
</project>

View File

@@ -0,0 +1,116 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>spring-cloud-kubernetes-integration-tests</artifactId>
<groupId>org.springframework.cloud</groupId>
<version>3.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>spring-cloud-kubernetes-client-catalog-watcher</artifactId>
<dependencies>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-kubernetes-client-discovery</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-kubernetes-test-support</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-webflux</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-transport-httpclient5</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>../src/main/resources</directory>
<filtering>true</filtering>
</resource>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<!-- build image in the 'package' phase, and ignore plain tests -->
<!-- via maven-surefire-plugin::skipTests -->
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<imageName>docker.io/springcloud/${project.artifactId}:${project.version}</imageName>
<imageBuilder>paketobuildpacks/builder</imageBuilder>
</configuration>
<executions>
<execution>
<id>build-image</id>
<configuration>
<skip>${skip.build.image}</skip>
</configuration>
<phase>package</phase>
<goals>
<goal>build-image</goal>
</goals>
</execution>
<execution>
<id>repackage</id>
<phase>package</phase>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- ignore plain tests (in the 'test' phase), so that we could build the image first, see above -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
<!-- run tests in the 'integration-tests' phase, one that is after 'package' (where we build the image) -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>integration-test</goal>
</goals>
</execution>
</executions>
<configuration>
<includes>
<include>${testsToRun}</include>
</includes>
</configuration>
</plugin>
</plugins>
</build>
</project>

View File

@@ -0,0 +1,34 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.catalog;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
/**
* @author wind57
*/
@SpringBootApplication
@EnableScheduling
public class Application {
public static void main(String[] args) {
SpringApplication.run(Application.class, args);
}
}

View File

@@ -0,0 +1,43 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.catalog;
import java.util.List;
import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace;
import org.springframework.stereotype.Service;
/**
* holds an EndpointNameAndNamespace object, needed so that the controller can poll to see
* if anything has changed. And we call the controller from our tests.
*
* @author wind57
*/
@Service
public class EndpointNameAndNamespaceService {
private List<EndpointNameAndNamespace> result;
public List<EndpointNameAndNamespace> result() {
return result;
}
public void setResult(List<EndpointNameAndNamespace> result) {
this.result = result;
}
}

View File

@@ -0,0 +1,50 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.catalog;
import java.util.List;
import org.springframework.cloud.client.discovery.event.HeartbeatEvent;
import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace;
import org.springframework.context.ApplicationEvent;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
/**
* Listener that will catch events from KubernetesCatalogWatch.
*
* @author wind57
*/
@Component
public class HeartBeatListener implements ApplicationListener<ApplicationEvent> {
private final EndpointNameAndNamespaceService service;
public HeartBeatListener(EndpointNameAndNamespaceService service) {
this.service = service;
}
@SuppressWarnings("unchecked")
@Override
public void onApplicationEvent(ApplicationEvent event) {
if (event instanceof HeartbeatEvent heartbeatEvent) {
List<EndpointNameAndNamespace> result = (List<EndpointNameAndNamespace>) heartbeatEvent.getValue();
service.setResult(result);
}
}
}

View File

@@ -0,0 +1,39 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.catalog;
import java.util.List;
import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
public class HeartbeatController {
private final EndpointNameAndNamespaceService service;
public HeartbeatController(EndpointNameAndNamespaceService service) {
this.service = service;
}
@GetMapping("/result")
public List<EndpointNameAndNamespace> result() {
return service.result();
}
}

View File

@@ -0,0 +1,304 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.catalog;
import java.time.Duration;
import java.util.List;
import java.util.Objects;
import io.kubernetes.client.openapi.apis.AppsV1Api;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.apis.NetworkingV1Api;
import io.kubernetes.client.openapi.models.V1Deployment;
import io.kubernetes.client.openapi.models.V1Ingress;
import io.kubernetes.client.openapi.models.V1Service;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.testcontainers.k3s.K3sContainer;
import reactor.netty.http.client.HttpClient;
import reactor.util.retry.Retry;
import reactor.util.retry.RetryBackoffSpec;
import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace;
import org.springframework.cloud.kubernetes.integration.tests.commons.Commons;
import org.springframework.cloud.kubernetes.integration.tests.commons.K8SUtils;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.core.ResolvableType;
import org.springframework.http.HttpMethod;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.web.reactive.function.client.WebClient;
import static org.awaitility.Awaitility.await;
import static org.springframework.cloud.kubernetes.integration.tests.commons.K8SUtils.createApiClient;
/**
* @author wind57
*/
class KubernetesClientCatalogWatchIT {
private static final String APP_NAME = "spring-cloud-kubernetes-client-catalog-watcher";
private static final String NAMESPACE = "default";
private static final K3sContainer K3S = Commons.container();
private static CoreV1Api api;
private static AppsV1Api appsApi;
private static NetworkingV1Api networkingApi;
private static K8SUtils k8SUtils;
private static String busyboxServiceName;
private static String busyboxDeploymentName;
private static String appDeploymentName;
private static String appServiceName;
private static String appIngressName;
@BeforeAll
static void beforeAll() throws Exception {
K3S.start();
Commons.validateImage(APP_NAME, K3S);
Commons.loadSpringCloudKubernetesImage(APP_NAME, K3S);
createApiClient(K3S.getKubeConfigYaml());
api = new CoreV1Api();
appsApi = new AppsV1Api();
networkingApi = new NetworkingV1Api();
k8SUtils = new K8SUtils(api, appsApi);
k8SUtils.setUp(NAMESPACE);
}
@BeforeEach
void beforeEach() throws Exception {
deployBusyboxManifests();
}
@AfterEach
void afterEach() throws Exception {
deleteApp();
}
/**
* <pre>
* - we deploy a busybox service with 2 replica pods
* - we receive an event from KubernetesCatalogWatcher, assert what is inside it
* - delete the busybox service
* - assert that we receive only spring-cloud-kubernetes-client-catalog-watcher pod
* </pre>
*/
@Test
void testCatalogWatchWithEndpoints() throws Exception {
deployApp(false);
assertLogStatement("stateGenerator is of type: KubernetesEndpointsCatalogWatch");
test();
}
@Test
void testCatalogWatchWithEndpointSlices() throws Exception {
deployApp(true);
assertLogStatement("stateGenerator is of type: KubernetesEndpointSlicesCatalogWatch");
test();
}
/**
* we log in debug mode the type of the StateGenerator we use, be that Endpoints or
* EndpointSlices. Here we make sure that in the test we actually use the correct
* type.
*/
private void assertLogStatement(String log) throws Exception {
String appPodName = K3S.execInContainer("kubectl", "get", "pods", "-l",
"app=spring-cloud-kubernetes-client-catalog-watcher", "-o=name", "--no-headers").getStdout();
String allLogs = K3S.execInContainer("kubectl", "logs", appPodName.trim()).getStdout();
Assertions.assertTrue(allLogs.contains(log));
}
/**
* the test is the same for both endpoints and endpoint slices, the set-up for them is
* different.
*/
@SuppressWarnings("unchecked")
private void test() throws Exception {
WebClient client = builder().baseUrl("localhost/result").build();
EndpointNameAndNamespace[] holder = new EndpointNameAndNamespace[2];
ResolvableType resolvableType = ResolvableType.forClassWithGenerics(List.class, EndpointNameAndNamespace.class);
await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(240)).until(() -> {
List<EndpointNameAndNamespace> result = (List<EndpointNameAndNamespace>) client.method(HttpMethod.GET)
.retrieve().bodyToMono(ParameterizedTypeReference.forType(resolvableType.getType()))
.retryWhen(retrySpec()).block();
// we get 3 pods as input, but because they are sorted by name in the catalog
// watcher implementation
// we will get the first busybox instances here.
if (result != null) {
holder[0] = result.get(0);
holder[1] = result.get(1);
return true;
}
return false;
});
EndpointNameAndNamespace resultOne = holder[0];
EndpointNameAndNamespace resultTwo = holder[1];
Assertions.assertNotNull(resultOne);
Assertions.assertNotNull(resultTwo);
Assertions.assertTrue(resultOne.endpointName().contains("busybox"));
Assertions.assertTrue(resultTwo.endpointName().contains("busybox"));
Assertions.assertEquals("default", resultOne.namespace());
Assertions.assertEquals("default", resultTwo.namespace());
deleteBusyboxApp();
// what we get after delete
EndpointNameAndNamespace[] afterDelete = new EndpointNameAndNamespace[1];
await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(240)).until(() -> {
List<EndpointNameAndNamespace> result = (List<EndpointNameAndNamespace>) client.method(HttpMethod.GET)
.retrieve().bodyToMono(ParameterizedTypeReference.forType(resolvableType.getType()))
.retryWhen(retrySpec()).block();
// we need to get the event from KubernetesCatalogWatch, but that happens
// on periodic bases. So in order to be sure we got the event we care about
// we wait until the result has a single entry, which means busybox was
// deleted
// + KubernetesCatalogWatch received the new update.
if (result != null && result.size() != 1) {
return false;
}
// we will only receive one pod here, our own
if (result != null) {
afterDelete[0] = result.get(0);
return true;
}
return false;
});
Assertions.assertTrue(afterDelete[0].endpointName().contains(APP_NAME));
Assertions.assertEquals("default", afterDelete[0].namespace());
}
private void deployBusyboxManifests() throws Exception {
V1Deployment busyboxDeployment = (V1Deployment) K8SUtils.readYamlFromClasspath(getBusyboxDeployment());
String[] image = K8SUtils.getImageFromDeployment(busyboxDeployment).split(":");
Commons.pullImage(image[0], image[1], K3S);
Commons.loadImage(image[0], image[1], "busybox", K3S);
appsApi.createNamespacedDeployment(NAMESPACE, busyboxDeployment, null, null, null, null);
busyboxDeploymentName = busyboxDeployment.getMetadata().getName();
V1Service busyboxService = (V1Service) K8SUtils.readYamlFromClasspath(getBusyboxService());
busyboxServiceName = busyboxService.getMetadata().getName();
api.createNamespacedService(NAMESPACE, busyboxService, null, null, null, null);
k8SUtils.waitForDeployment(busyboxDeploymentName, NAMESPACE);
}
private static void deployApp(boolean useEndpointSlices) throws Exception {
V1Deployment appDeployment = useEndpointSlices
? (V1Deployment) K8SUtils.readYamlFromClasspath(getEndpointSlicesAppDeployment())
: (V1Deployment) K8SUtils.readYamlFromClasspath(getEndpointsAppDeployment());
String version = K8SUtils.getPomVersion();
String currentImage = appDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).getImage();
appDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).setImage(currentImage + ":" + version);
appsApi.createNamespacedDeployment(NAMESPACE, appDeployment, null, null, null, null);
appDeploymentName = appDeployment.getMetadata().getName();
V1Service appService = (V1Service) K8SUtils.readYamlFromClasspath(getAppService());
appServiceName = appService.getMetadata().getName();
api.createNamespacedService(NAMESPACE, appService, null, null, null, null);
k8SUtils.waitForDeployment(appDeploymentName, NAMESPACE);
V1Ingress appIngress = (V1Ingress) K8SUtils.readYamlFromClasspath(getAppIngress());
appIngressName = appIngress.getMetadata().getName();
networkingApi.createNamespacedIngress(NAMESPACE, appIngress, null, null, null, null);
k8SUtils.waitForIngress(appIngressName, NAMESPACE);
}
private void deleteBusyboxApp() throws Exception {
appsApi.deleteNamespacedDeployment(busyboxDeploymentName, NAMESPACE, null, null, null, null, null, null);
api.deleteNamespacedService(busyboxServiceName, NAMESPACE, null, null, null, null, null, null);
k8SUtils.waitForDeploymentToBeDeleted(busyboxDeploymentName, NAMESPACE);
}
private void deleteApp() throws Exception {
appsApi.deleteNamespacedDeployment(appDeploymentName, NAMESPACE, null, null, null, null, null, null);
api.deleteNamespacedService(appServiceName, NAMESPACE, null, null, null, null, null, null);
networkingApi.deleteNamespacedIngress(appIngressName, NAMESPACE, null, null, null, null, null, null);
k8SUtils.waitForDeploymentToBeDeleted(busyboxDeploymentName, NAMESPACE);
}
private static String getBusyboxService() {
return "busybox/service.yaml";
}
private static String getBusyboxDeployment() {
return "busybox/deployment.yaml";
}
/**
* deployment where support for endpoint slices is equal to false
*/
private static String getEndpointsAppDeployment() {
return "app/watcher-endpoints-deployment.yaml";
}
private static String getEndpointSlicesAppDeployment() {
return "app/watcher-endpoint-slices-deployment.yaml";
}
private static String getAppIngress() {
return "app/watcher-ingress.yaml";
}
private static String getAppService() {
return "app/watcher-service.yaml";
}
private WebClient.Builder builder() {
return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create()));
}
private RetryBackoffSpec retrySpec() {
return Retry.fixedDelay(15, Duration.ofSeconds(1)).filter(Objects::nonNull);
}
}

View File

@@ -0,0 +1,353 @@
/*
* Copyright 2013-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.client.catalog;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import io.kubernetes.client.openapi.apis.AppsV1Api;
import io.kubernetes.client.openapi.apis.CoreV1Api;
import io.kubernetes.client.openapi.apis.NetworkingV1Api;
import io.kubernetes.client.openapi.models.V1Deployment;
import io.kubernetes.client.openapi.models.V1EnvVar;
import io.kubernetes.client.openapi.models.V1EnvVarBuilder;
import io.kubernetes.client.openapi.models.V1Ingress;
import io.kubernetes.client.openapi.models.V1NamespaceBuilder;
import io.kubernetes.client.openapi.models.V1Service;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.testcontainers.k3s.K3sContainer;
import reactor.netty.http.client.HttpClient;
import reactor.util.retry.Retry;
import reactor.util.retry.RetryBackoffSpec;
import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace;
import org.springframework.cloud.kubernetes.integration.tests.commons.Commons;
import org.springframework.cloud.kubernetes.integration.tests.commons.K8SUtils;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.core.ResolvableType;
import org.springframework.http.HttpMethod;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.web.reactive.function.client.WebClient;
import static org.awaitility.Awaitility.await;
import static org.springframework.cloud.kubernetes.integration.tests.commons.K8SUtils.createApiClient;
public class KubernetesClientCatalogWatchNamespacesIT {
private static final String APP_NAME = "spring-cloud-kubernetes-client-catalog-watcher";
private static final String NAMESPACE_A = "namespacea";
private static final String NAMESPACE_B = "namespaceb";
private static final String NAMESPACE_DEFAULT = "default";
private static final K3sContainer K3S = Commons.container();
private static CoreV1Api api;
private static AppsV1Api appsApi;
private static NetworkingV1Api networkingApi;
private static K8SUtils k8SUtils;
private static String busyboxServiceNameA;
private static String busyboxServiceNameB;
private static String busyboxDeploymentNameA;
private static String busyboxDeploymentNameB;
private static String appDeploymentName;
private static String appServiceName;
private static String appIngressName;
@BeforeAll
static void beforeAll() throws Exception {
K3S.start();
Commons.validateImage(APP_NAME, K3S);
Commons.loadSpringCloudKubernetesImage(APP_NAME, K3S);
createApiClient(K3S.getKubeConfigYaml());
api = new CoreV1Api();
appsApi = new AppsV1Api();
networkingApi = new NetworkingV1Api();
k8SUtils = new K8SUtils(api, appsApi);
k8SUtils.setUp(NAMESPACE_DEFAULT);
}
@BeforeEach
void beforeEach() throws Exception {
api.createNamespace(new V1NamespaceBuilder().withNewMetadata().withName(NAMESPACE_A).and().build(), null, null,
null, null);
api.createNamespace(new V1NamespaceBuilder().withNewMetadata().withName(NAMESPACE_B).and().build(), null, null,
null, null);
k8SUtils.setUpClusterWide(NAMESPACE_DEFAULT, Set.of(NAMESPACE_A, NAMESPACE_B));
deployBusyboxManifests();
}
@AfterEach
void afterEach() throws Exception {
deleteApp();
k8SUtils.deleteNamespace(NAMESPACE_A);
k8SUtils.deleteNamespace(NAMESPACE_B);
}
/**
* <pre>
* - we deploy one busybox service with 2 replica pods in namespace namespacea
* - we deploy one busybox service with 2 replica pods in namespace namespaceb
* - we enable the search to be made in namespacea and default ones
* - we receive an event from KubernetesCatalogWatcher, assert what is inside it
* - delete both busybox services in namespacea and namespaceb
* - assert that we receive only spring-cloud-kubernetes-client-catalog-watcher pod
* </pre>
*/
@Test
void testCatalogWatchWithEndpoints() throws Exception {
deployApp(false);
assertLogStatement("stateGenerator is of type: KubernetesEndpointsCatalogWatch");
test();
}
@Test
void testCatalogWatchWithEndpointSlices() throws Exception {
deployApp(true);
assertLogStatement("stateGenerator is of type: KubernetesEndpointSlicesCatalogWatch");
test();
}
/**
* we log in debug mode the type of the StateGenerator we use, be that Endpoints or
* EndpointSlices. Here we make sure that in the test we actually use the correct
* type.
*/
private void assertLogStatement(String log) throws Exception {
String appPodName = K3S
.execInContainer("kubectl", "get", "pods", "-l",
"app=spring-cloud-kubernetes-client-catalog-watcher", "-o=name", "--no-headers")
.getStdout();
String allLogs = K3S.execInContainer("kubectl", "logs", appPodName.trim()).getStdout();
Assertions.assertTrue(allLogs.contains(log));
}
/**
* the test is the same for both endpoints and endpoint slices, the set-up for them is
* different.
*/
@SuppressWarnings("unchecked")
private void test() throws Exception {
WebClient client = builder().baseUrl("localhost/result").build();
EndpointNameAndNamespace[] holder = new EndpointNameAndNamespace[2];
ResolvableType resolvableType = ResolvableType.forClassWithGenerics(List.class, EndpointNameAndNamespace.class);
await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(240)).until(() -> {
List<EndpointNameAndNamespace> result = (List<EndpointNameAndNamespace>) client.method(HttpMethod.GET)
.retrieve().bodyToMono(ParameterizedTypeReference.forType(resolvableType.getType()))
.retryWhen(retrySpec()).block();
// we get 3 pods as input, but because they are sorted by name in the catalog
// watcher implementation
// we will get the first busybox instances here.
if (result != null) {
holder[0] = result.get(0);
holder[1] = result.get(1);
return true;
}
return false;
});
EndpointNameAndNamespace resultOne = holder[0];
EndpointNameAndNamespace resultTwo = holder[1];
Assertions.assertNotNull(resultOne);
Assertions.assertNotNull(resultTwo);
Assertions.assertTrue(resultOne.endpointName().contains("busybox"));
Assertions.assertTrue(resultTwo.endpointName().contains("busybox"));
Assertions.assertEquals(NAMESPACE_A, resultOne.namespace());
Assertions.assertEquals(NAMESPACE_A, resultTwo.namespace());
deleteBusyboxApp();
// what we get after delete
EndpointNameAndNamespace[] afterDelete = new EndpointNameAndNamespace[1];
await().pollInterval(Duration.ofSeconds(1)).atMost(Duration.ofSeconds(240)).until(() -> {
List<EndpointNameAndNamespace> result = (List<EndpointNameAndNamespace>) client.method(HttpMethod.GET)
.retrieve().bodyToMono(ParameterizedTypeReference.forType(resolvableType.getType()))
.retryWhen(retrySpec()).block();
// we need to get the event from KubernetesCatalogWatch, but that happens
// on periodic bases. So in order to be sure we got the event we care about
// we wait until the result has a single entry, which means busybox was
// deleted
// + KubernetesCatalogWatch received the new update.
if (result != null && result.size() != 1) {
return false;
}
// we will only receive one pod here, our own
if (result != null) {
afterDelete[0] = result.get(0);
return true;
}
return false;
});
Assertions.assertTrue(afterDelete[0].endpointName().contains(APP_NAME));
Assertions.assertEquals("default", afterDelete[0].namespace());
}
private void deployBusyboxManifests() throws Exception {
V1Deployment busyboxDeployment = (V1Deployment) K8SUtils.readYamlFromClasspath(getBusyboxDeployment());
String[] image = K8SUtils.getImageFromDeployment(busyboxDeployment).split(":");
Commons.pullImage(image[0], image[1], K3S);
Commons.loadImage(image[0], image[1], "busybox", K3S);
// namespace_a
appsApi.createNamespacedDeployment(NAMESPACE_A, busyboxDeployment, null, null, null, null);
busyboxDeploymentNameA = busyboxDeployment.getMetadata().getName();
V1Service busyboxServiceA = (V1Service) K8SUtils.readYamlFromClasspath(getBusyboxService());
busyboxServiceNameA = busyboxServiceA.getMetadata().getName();
api.createNamespacedService(NAMESPACE_A, busyboxServiceA, null, null, null, null);
k8SUtils.waitForDeployment(busyboxDeploymentNameA, NAMESPACE_A);
// namespace_b
appsApi.createNamespacedDeployment(NAMESPACE_B, busyboxDeployment, null, null, null, null);
busyboxDeploymentNameB = busyboxDeployment.getMetadata().getName();
V1Service busyboxServiceB = (V1Service) K8SUtils.readYamlFromClasspath(getBusyboxService());
busyboxServiceNameB = busyboxServiceB.getMetadata().getName();
api.createNamespacedService(NAMESPACE_B, busyboxServiceB, null, null, null, null);
k8SUtils.waitForDeployment(busyboxDeploymentNameA, NAMESPACE_A);
}
private static void deployApp(boolean useEndpointSlices) throws Exception {
V1Deployment appDeployment = useEndpointSlices
? (V1Deployment) K8SUtils.readYamlFromClasspath(getEndpointSlicesAppDeployment())
: (V1Deployment) K8SUtils.readYamlFromClasspath(getEndpointsAppDeployment());
String version = K8SUtils.getPomVersion();
String currentImage = appDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).getImage();
appDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).setImage(currentImage + ":" + version);
List<V1EnvVar> envVars = new ArrayList<>(
appDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv());
V1EnvVar namespaceAEnvVar = new V1EnvVarBuilder().withName("SPRING_CLOUD_KUBERNETES_DISCOVERY_NAMESPACES_0")
.withValue(NAMESPACE_A).build();
V1EnvVar namespaceDefaultEnvVar = new V1EnvVarBuilder()
.withName("SPRING_CLOUD_KUBERNETES_DISCOVERY_NAMESPACES_1").withValue(NAMESPACE_DEFAULT).build();
envVars.add(namespaceAEnvVar);
envVars.add(namespaceDefaultEnvVar);
appDeployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(envVars);
appsApi.createNamespacedDeployment(NAMESPACE_DEFAULT, appDeployment, null, null, null, null);
appDeploymentName = appDeployment.getMetadata().getName();
V1Service appService = (V1Service) K8SUtils.readYamlFromClasspath(getAppService());
appServiceName = appService.getMetadata().getName();
api.createNamespacedService(NAMESPACE_DEFAULT, appService, null, null, null, null);
k8SUtils.waitForDeployment(appDeploymentName, NAMESPACE_DEFAULT);
V1Ingress appIngress = (V1Ingress) K8SUtils.readYamlFromClasspath(getAppIngress());
appIngressName = appIngress.getMetadata().getName();
networkingApi.createNamespacedIngress(NAMESPACE_DEFAULT, appIngress, null, null, null, null);
k8SUtils.waitForIngress(appIngressName, NAMESPACE_DEFAULT);
}
private void deleteBusyboxApp() throws Exception {
// namespacea
appsApi.deleteNamespacedDeployment(busyboxDeploymentNameA, NAMESPACE_A, null, null, null, null, null, null);
api.deleteNamespacedService(busyboxServiceNameA, NAMESPACE_A, null, null, null, null, null, null);
k8SUtils.waitForDeploymentToBeDeleted(busyboxDeploymentNameA, NAMESPACE_A);
// namespaceb
appsApi.deleteNamespacedDeployment(busyboxDeploymentNameB, NAMESPACE_B, null, null, null, null, null, null);
api.deleteNamespacedService(busyboxServiceNameB, NAMESPACE_B, null, null, null, null, null, null);
k8SUtils.waitForDeploymentToBeDeleted(busyboxDeploymentNameB, NAMESPACE_B);
}
private void deleteApp() throws Exception {
appsApi.deleteNamespacedDeployment(appDeploymentName, NAMESPACE_DEFAULT, null, null, null, null, null, null);
api.deleteNamespacedService(appServiceName, NAMESPACE_DEFAULT, null, null, null, null, null, null);
networkingApi.deleteNamespacedIngress(appIngressName, NAMESPACE_DEFAULT, null, null, null, null, null, null);
}
private static String getBusyboxService() {
return "busybox/service.yaml";
}
private static String getBusyboxDeployment() {
return "busybox/deployment.yaml";
}
/**
* deployment where support for endpoint slices is equal to false
*/
private static String getEndpointsAppDeployment() {
return "app/watcher-endpoints-deployment.yaml";
}
private static String getEndpointSlicesAppDeployment() {
return "app/watcher-endpoint-slices-deployment.yaml";
}
private static String getAppIngress() {
return "app/watcher-ingress.yaml";
}
private static String getAppService() {
return "app/watcher-service.yaml";
}
private WebClient.Builder builder() {
return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create()));
}
private RetryBackoffSpec retrySpec() {
return Retry.fixedDelay(15, Duration.ofSeconds(1)).filter(Objects::nonNull);
}
}

View File

@@ -0,0 +1,33 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: spring-cloud-kubernetes-client-catalog-watcher
spec:
selector:
matchLabels:
app: spring-cloud-kubernetes-client-catalog-watcher
template:
metadata:
labels:
app: spring-cloud-kubernetes-client-catalog-watcher
spec:
serviceAccountName: spring-cloud-kubernetes-serviceaccount
containers:
- name: spring-cloud-kubernetes-client-catalog-watcher
image: docker.io/springcloud/spring-cloud-kubernetes-client-catalog-watcher
imagePullPolicy: IfNotPresent
readinessProbe:
httpGet:
port: 8080
path: /actuator/health/readiness
livenessProbe:
httpGet:
port: 8080
path: /actuator/health/liveness
ports:
- containerPort: 8080
env:
- name: LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_CLIENT_DISCOVERY_CATALOG
value: DEBUG
- name: SPRING_CLOUD_KUBERNETES_DISCOVERY_USE_ENDPOINT_SLICES
value: true

View File

@@ -0,0 +1,33 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: spring-cloud-kubernetes-client-catalog-watcher
spec:
selector:
matchLabels:
app: spring-cloud-kubernetes-client-catalog-watcher
template:
metadata:
labels:
app: spring-cloud-kubernetes-client-catalog-watcher
spec:
serviceAccountName: spring-cloud-kubernetes-serviceaccount
containers:
- name: spring-cloud-kubernetes-client-catalog-watcher
image: docker.io/springcloud/spring-cloud-kubernetes-client-catalog-watcher
imagePullPolicy: IfNotPresent
readinessProbe:
httpGet:
port: 8080
path: /actuator/health/readiness
livenessProbe:
httpGet:
port: 8080
path: /actuator/health/liveness
ports:
- containerPort: 8080
env:
- name: LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_CLIENT_DISCOVERY_CATALOG
value: DEBUG
- name: SPRING_CLOUD_KUBERNETES_DISCOVERY_USE_ENDPOINT_SLICES
value: false

View File

@@ -0,0 +1,16 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: spring-cloud-kubernetes-client-catalog-watcher-ingress
namespace: default
spec:
rules:
- http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: spring-cloud-kubernetes-client-catalog-watcher-service
port:
number: 8080

View File

@@ -0,0 +1,14 @@
apiVersion: v1
kind: Service
metadata:
labels:
app: spring-cloud-kubernetes-client-catalog-watcher-service
name: spring-cloud-kubernetes-client-catalog-watcher-service
spec:
ports:
- name: http
port: 8080
targetPort: 8080
selector:
app: spring-cloud-kubernetes-client-catalog-watcher
type: ClusterIP

View File

@@ -0,0 +1,22 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: busybox
spec:
selector:
matchLabels:
app: busybox
version: v1
replicas: 2
template:
metadata:
labels:
app: busybox
version: v1
spec:
containers:
- name: busybox
# image: arm64/busybox:latest
image: busybox:1.35
command: ["/bin/sh"]
args: ["-c", "sleep 100000"]

View File

@@ -0,0 +1,12 @@
apiVersion: v1
kind: Service
metadata:
name: busybox-service
spec:
selector:
app: busybox
type: ClusterIP
ports:
- name: busybox-port
port: 8080
targetPort: 80

View File

@@ -0,0 +1,15 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="STDOUT"/>
</root>
<logger name="org.testcontainers" level="INFO"/>
<logger name="com.github.dockerjava" level="WARN"/>
<logger name="io.fabric8.kubernetes.client" level="ERROR"/>
</configuration>

View File

@@ -57,7 +57,7 @@ import static org.awaitility.Awaitility.await;
/**
* @author wind57
*/
class CatalogWatchWithNamespacesIT {
class Fabric8CatalogWatchWithNamespacesIT {
private static final String APP_NAME = "spring-cloud-kubernetes-fabric8-client-catalog-watcher";

View File

@@ -309,6 +309,14 @@ public class K8SUtils {
() -> rbacApi.createNamespacedRole(namespace, role, null, null, null, null));
}
public void deleteNamespace(String name) throws Exception {
api.deleteNamespace(name, null, null, null, null, null, null);
await().pollInterval(Duration.ofSeconds(1)).atMost(30, TimeUnit.SECONDS)
.until(() -> api.listNamespace(null, null, null, null, null, null, null, null, null, null).getItems()
.stream().noneMatch(x -> x.getMetadata().getName().equals(name)));
}
public void setUpClusterWide(String serviceAccountNamespace, Set<String> namespaces) throws Exception {
V1ServiceAccount serviceAccount = getConfigK8sClientItClusterServiceAccount();