diff --git a/docs/src/main/asciidoc/discovery-client.adoc b/docs/src/main/asciidoc/discovery-client.adoc index 53b8df5e..99d6a182 100644 --- a/docs/src/main/asciidoc/discovery-client.adoc +++ b/docs/src/main/asciidoc/discovery-client.adoc @@ -116,6 +116,10 @@ By default all of the ports and their names will be added to the metadata of the As said before, if you want to get the list of `ServiceInstance` to also include the `ExternalName` type services, you need to enable that support via: `spring.cloud.kubernetes.discovery.include-external-name-services=true`. As such, when calling `DiscoveryClient::getInstances` those will be returned also. You can distinguish between `ExternalName` and any other types by inspecting `ServiceInstance::getMetadata` and lookup for a field called `type`. This will be the type of the service returned : `ExternalName`/`ClusterIP`, etc. +`ServiceInstance` can include the labels and annotations of specific pods from the underlying service instance. To obtain such information, you need to also enable: + +`spring.cloud.kubernetes.discovery.metadata.add-pod-labels=true` and/or `spring.cloud.kubernetes.discovery.metadata.add-pod-annotations=true`. At the moment, such functionality is present only in the fabric8 client implementation, but will be added to the kubernetes native client in a later release. + If, for any reason, you need to disable the `DiscoveryClient`, you can set the following property in `application.properties`: ==== diff --git a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/DefaultKubernetesServiceInstance.java b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/DefaultKubernetesServiceInstance.java index e73342c7..f9d4c729 100644 --- a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/DefaultKubernetesServiceInstance.java +++ b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/DefaultKubernetesServiceInstance.java @@ -35,8 +35,8 @@ import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesD * @param cluster the cluster the service resides in. */ public record DefaultKubernetesServiceInstance(String instanceId, String serviceId, String host, int port, - Map metadata, boolean secure, String namespace, - String cluster) implements KubernetesServiceInstance { + Map metadata, boolean secure, String namespace, String cluster, + Map> podMetadata) implements KubernetesServiceInstance { /** * @param instanceId the id of the instance. @@ -48,7 +48,12 @@ public record DefaultKubernetesServiceInstance(String instanceId, String service */ public DefaultKubernetesServiceInstance(String instanceId, String serviceId, String host, int port, Map metadata, boolean secure) { - this(instanceId, serviceId, host, port, metadata, secure, null, null); + this(instanceId, serviceId, host, port, metadata, secure, null, null, Map.of()); + } + + public DefaultKubernetesServiceInstance(String instanceId, String serviceId, String host, int port, + Map metadata, boolean secure, String namespace, String cluster) { + this(instanceId, serviceId, host, port, metadata, secure, namespace, cluster, Map.of()); } @Override @@ -101,6 +106,11 @@ public record DefaultKubernetesServiceInstance(String instanceId, String service return this.cluster; } + @Override + public Map> podMetadata() { + return podMetadata; + } + private URI createUri(String scheme, String host, int port) { // assume ExternalName type of service if (port == -1) { diff --git a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryProperties.java b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryProperties.java index 69d3b101..f5f0dfba 100644 --- a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryProperties.java +++ b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryProperties.java @@ -92,15 +92,30 @@ public record KubernetesDiscoveryProperties( * @param annotationsPrefix prefix for the annotations * @param addPorts include ports as metadata * @param portsPrefix prefix for the ports, by default it is "port." + * @param addPodLabels add pod labels as part of the response. + * @param addPodAnnotations add pod annotations as part of the response. */ public record Metadata(@DefaultValue("true") boolean addLabels, String labelsPrefix, @DefaultValue("true") boolean addAnnotations, String annotationsPrefix, - @DefaultValue("true") boolean addPorts, @DefaultValue("port.") String portsPrefix) { + @DefaultValue("true") boolean addPorts, @DefaultValue("port.") String portsPrefix, boolean addPodLabels, + boolean addPodAnnotations) { + + @ConstructorBinding + public Metadata { + + } + + public Metadata(@DefaultValue("true") boolean addLabels, String labelsPrefix, + @DefaultValue("true") boolean addAnnotations, String annotationsPrefix, + @DefaultValue("true") boolean addPorts, @DefaultValue("port.") String portsPrefix) { + + this(addLabels, labelsPrefix, addAnnotations, annotationsPrefix, addPorts, portsPrefix, false, false); + } /** * Default instance. */ - public static final Metadata DEFAULT = new Metadata(true, null, true, null, true, "port."); + public static final Metadata DEFAULT = new Metadata(true, null, true, null, true, "port.", false, false); } diff --git a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesServiceInstance.java b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesServiceInstance.java index 2f6c4e20..53832de7 100644 --- a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesServiceInstance.java +++ b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesServiceInstance.java @@ -16,6 +16,8 @@ package org.springframework.cloud.kubernetes.commons.discovery; +import java.util.Map; + import org.springframework.cloud.client.ServiceInstance; /** @@ -29,4 +31,8 @@ sealed public interface KubernetesServiceInstance extends ServiceInstance permit String getCluster(); + default Map> podMetadata() { + return Map.of(); + } + } diff --git a/spring-cloud-kubernetes-commons/src/test/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryPropertiesMetadataTests.java b/spring-cloud-kubernetes-commons/src/test/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryPropertiesMetadataTests.java index b01f410a..a5d645d5 100644 --- a/spring-cloud-kubernetes-commons/src/test/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryPropertiesMetadataTests.java +++ b/spring-cloud-kubernetes-commons/src/test/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryPropertiesMetadataTests.java @@ -39,18 +39,24 @@ class KubernetesDiscoveryPropertiesMetadataTests { assertThat(m.annotationsPrefix()).isNull(); assertThat(m.addPorts()).isTrue(); assertThat(m.portsPrefix()).isEqualTo("port."); + assertThat(m.addPodLabels()).isFalse(); + assertThat(m.addPodAnnotations()).isFalse(); } @Test void testSpringBindingFields() { new ApplicationContextRunner().withUserConfiguration(Config.class) - .withPropertyValues("spring.cloud.kubernetes.discovery.metadata.labelsPrefix=labelsPrefix") + .withPropertyValues("spring.cloud.kubernetes.discovery.metadata.labelsPrefix=labelsPrefix", + "spring.cloud.kubernetes.discovery.metadata.add-pod-annotations=true", + "spring.cloud.kubernetes.discovery.metadata.add-pod-labels=true") .run(context -> { KubernetesDiscoveryProperties props = context.getBean(KubernetesDiscoveryProperties.class); assertThat(props).isNotNull(); assertThat(props.metadata().labelsPrefix()).isEqualTo("labelsPrefix"); assertThat(props.metadata().addPorts()).isTrue(); assertThat(props.metadata().portsPrefix()).isEqualTo("port."); + assertThat(props.metadata().addPodLabels()).isTrue(); + assertThat(props.metadata().addPodAnnotations()).isTrue(); }); } diff --git a/spring-cloud-kubernetes-commons/src/test/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesServiceInstanceTests.java b/spring-cloud-kubernetes-commons/src/test/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesServiceInstanceTests.java index 9a06cafe..2e13e6e6 100644 --- a/spring-cloud-kubernetes-commons/src/test/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesServiceInstanceTests.java +++ b/spring-cloud-kubernetes-commons/src/test/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesServiceInstanceTests.java @@ -44,6 +44,7 @@ class KubernetesServiceInstanceTests { assertThat(instance.getScheme()).isEqualTo("https"); assertThat(instance.getNamespace()).isEqualTo("spring-k8s"); assertThat(instance.getCluster()).isNull(); + assertThat(instance.podMetadata()).isEqualTo(Map.of()); } @Test @@ -61,6 +62,27 @@ class KubernetesServiceInstanceTests { assertThat(instance.getScheme()).isEqualTo("https"); assertThat(instance.getNamespace()).isEqualTo("spring-k8s"); assertThat(instance.getCluster()).isEqualTo("cluster"); + assertThat(instance.podMetadata()).isEqualTo(Map.of()); + } + + @Test + void testThirdConstructor() { + DefaultKubernetesServiceInstance instance = new DefaultKubernetesServiceInstance("instanceId", "serviceId", + "host", 8080, Map.of("a", "b"), true, "spring-k8s", "cluster", + Map.of("labels", Map.of("a", "b"), "annotations", Map.of("c", "d"))); + + assertThat(instance.getInstanceId()).isEqualTo("instanceId"); + assertThat(instance.getServiceId()).isEqualTo("serviceId"); + assertThat(instance.getHost()).isEqualTo("host"); + assertThat(instance.getPort()).isEqualTo(8080); + assertThat(instance.isSecure()).isTrue(); + assertThat(instance.getUri()).isEqualTo(URI.create("https://host:8080")); + assertThat(instance.getMetadata()).isEqualTo(Map.of("a", "b")); + assertThat(instance.getScheme()).isEqualTo("https"); + assertThat(instance.getNamespace()).isEqualTo("spring-k8s"); + assertThat(instance.getCluster()).isEqualTo("cluster"); + assertThat(instance.podMetadata()) + .isEqualTo(Map.of("labels", Map.of("a", "b"), "annotations", Map.of("c", "d"))); } @Test diff --git a/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClient.java b/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClient.java index 78c5557a..59322600 100644 --- a/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClient.java +++ b/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClient.java @@ -121,7 +121,7 @@ public class KubernetesDiscoveryClient implements DiscoveryClient, EnvironmentAw Map serviceMetadata = serviceMetadata(serviceId, service, properties, List.of(), service.getMetadata().getNamespace()); ServiceInstance externalNameServiceInstance = serviceInstance(null, service, null, -1, serviceId, - serviceMetadata, service.getMetadata().getNamespace()); + serviceMetadata, service.getMetadata().getNamespace(), properties, client); instances.add(externalNameServiceInstance); } } @@ -152,7 +152,7 @@ public class KubernetesDiscoveryClient implements DiscoveryClient, EnvironmentAw List addresses = addresses(endpointSubset, properties); for (EndpointAddress endpointAddress : addresses) { ServiceInstance serviceInstance = serviceInstance(servicePortSecureResolver, service, endpointAddress, - endpointPort, serviceId, serviceMetadata, namespace); + endpointPort, serviceId, serviceMetadata, namespace, properties, client); instances.add(serviceInstance); } } diff --git a/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClientUtils.java b/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClientUtils.java index 7cb5dd77..29e5a15a 100644 --- a/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClientUtils.java +++ b/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClientUtils.java @@ -29,7 +29,9 @@ import io.fabric8.kubernetes.api.model.EndpointPort; import io.fabric8.kubernetes.api.model.EndpointSubset; import io.fabric8.kubernetes.api.model.Endpoints; import io.fabric8.kubernetes.api.model.EndpointsList; +import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.ObjectReference; +import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.Service; import io.fabric8.kubernetes.api.model.ServiceList; import io.fabric8.kubernetes.client.KubernetesClient; @@ -51,6 +53,7 @@ import org.springframework.util.StringUtils; import static java.util.stream.Collectors.toMap; import static org.springframework.cloud.kubernetes.commons.config.ConfigUtils.keysWithPrefix; +import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.EXTERNAL_NAME; import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.HTTP; import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.HTTPS; import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.NAMESPACE_METADATA_KEY; @@ -236,7 +239,8 @@ final class KubernetesDiscoveryClientUtils { static ServiceInstance serviceInstance(@Nullable ServicePortSecureResolver servicePortSecureResolver, Service service, @Nullable EndpointAddress endpointAddress, int endpointPort, String serviceId, - Map serviceMetadata, String namespace) { + Map serviceMetadata, String namespace, KubernetesDiscoveryProperties properties, + KubernetesClient client) { // instanceId is usually the pod-uid as seen in the .metadata.uid String instanceId = Optional.ofNullable(endpointAddress).map(EndpointAddress::getTargetRef) .map(ObjectReference::getUid).orElseGet(() -> service.getMetadata().getUid()); @@ -254,8 +258,11 @@ final class KubernetesDiscoveryClientUtils { String host = Optional.ofNullable(endpointAddress).map(EndpointAddress::getIp) .orElseGet(() -> service.getSpec().getExternalName()); + Map> podMetadata = podMetadata(client, serviceMetadata, properties, endpointAddress, + namespace); + return new DefaultKubernetesServiceInstance(instanceId, serviceId, host, endpointPort, serviceMetadata, secured, - namespace, null); + namespace, null, podMetadata); } static List services(KubernetesDiscoveryProperties properties, KubernetesClient client, @@ -287,6 +294,37 @@ final class KubernetesDiscoveryClientUtils { return services; } + static Map> podMetadata(KubernetesClient client, Map serviceMetadata, + KubernetesDiscoveryProperties properties, EndpointAddress endpointAddress, String namespace) { + if (!EXTERNAL_NAME.equals(serviceMetadata.get(SERVICE_TYPE))) { + if (properties.metadata().addPodLabels() || properties.metadata().addPodAnnotations()) { + String podName = Optional.ofNullable(endpointAddress).map(EndpointAddress::getTargetRef) + .filter(objectReference -> "Pod".equals(objectReference.getKind())) + .map(ObjectReference::getName).orElse(null); + + if (podName != null) { + ObjectMeta metadata = Optional + .ofNullable(client.pods().inNamespace(namespace).withName(podName).get()) + .map(Pod::getMetadata).orElse(new ObjectMeta()); + Map> result = new HashMap<>(); + if (properties.metadata().addPodLabels() && !metadata.getLabels().isEmpty()) { + result.put("labels", metadata.getLabels()); + } + + if (properties.metadata().addPodAnnotations() && !metadata.getAnnotations().isEmpty()) { + result.put("annotations", metadata.getAnnotations()); + } + + LOG.debug(() -> "adding podMetadata : " + result + " from pod : " + podName); + return result; + } + + } + } + + return Map.of(); + } + /** * serviceName can be null, in which case, such a filter will not be applied. */ diff --git a/spring-cloud-kubernetes-fabric8-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClientTests.java b/spring-cloud-kubernetes-fabric8-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClientTests.java index 9287ef91..958c5c3e 100644 --- a/spring-cloud-kubernetes-fabric8-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClientTests.java +++ b/spring-cloud-kubernetes-fabric8-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClientTests.java @@ -20,19 +20,22 @@ import java.util.List; import java.util.Map; import java.util.Set; +import io.fabric8.kubernetes.api.model.EndpointAddressBuilder; +import io.fabric8.kubernetes.api.model.EndpointPortBuilder; +import io.fabric8.kubernetes.api.model.EndpointSubsetBuilder; import io.fabric8.kubernetes.api.model.Endpoints; import io.fabric8.kubernetes.api.model.EndpointsBuilder; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.api.model.ObjectReferenceBuilder; +import io.fabric8.kubernetes.api.model.PodBuilder; import io.fabric8.kubernetes.api.model.Service; import io.fabric8.kubernetes.api.model.ServiceBuilder; import io.fabric8.kubernetes.api.model.ServiceSpecBuilder; -import io.fabric8.kubernetes.client.Config; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -51,17 +54,6 @@ class KubernetesDiscoveryClientTests { private static KubernetesClient client; - @BeforeAll - static void setUp() { - // Configure the kubernetes master url to point to the mock server - System.setProperty(Config.KUBERNETES_MASTER_SYSTEM_PROPERTY, client.getConfiguration().getMasterUrl()); - System.setProperty(Config.KUBERNETES_TRUST_CERT_SYSTEM_PROPERTY, "true"); - System.setProperty(Config.KUBERNETES_AUTH_TRYKUBECONFIG_SYSTEM_PROPERTY, "false"); - System.setProperty(Config.KUBERNETES_AUTH_TRYSERVICEACCOUNT_SYSTEM_PROPERTY, "false"); - System.setProperty(Config.KUBERNETES_NAMESPACE_SYSTEM_PROPERTY, "test"); - System.setProperty(Config.KUBERNETES_HTTP2_DISABLE, "true"); - } - @AfterEach void afterEach() { client.endpoints().inAnyNamespace().delete(); @@ -527,6 +519,44 @@ class KubernetesDiscoveryClientTests { "labels-prefix-label-key", "label-value", "annotations-prefix-abc", "def", "type", "ExternalName")); } + @Test + void testPodMetadata() { + Service nonExternalNameService = new ServiceBuilder() + .withSpec(new ServiceSpecBuilder().withType("ClusterIP").build()).withNewMetadata() + .withName("blue-service").withNamespace("a").endMetadata().build(); + client.services().inNamespace("a").resource(nonExternalNameService).create(); + + client.endpoints().inNamespace("a").resource(new EndpointsBuilder() + .withMetadata(new ObjectMetaBuilder().withName("blue-service").build()) + .withSubsets(new EndpointSubsetBuilder().withPorts(new EndpointPortBuilder().withPort(8080).build()) + .withAddresses(new EndpointAddressBuilder().withIp("127.0.0.1") + .withTargetRef(new ObjectReferenceBuilder().withKind("Pod").withName("my-pod").build()) + .build()) + .build()) + .build()).create(); + + client.pods().inNamespace("a").resource(new PodBuilder().withMetadata(new ObjectMetaBuilder().withName("my-pod") + .withLabels(Map.of("a", "b")).withAnnotations(Map.of("c", "d")).build()).build()).create(); + + KubernetesDiscoveryProperties.Metadata metadata = new KubernetesDiscoveryProperties.Metadata(true, + "labels-prefix-", true, "annotations-prefix-", true, "ports-prefix", true, true); + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of("a", "b"), true, + 60L, false, "", Set.of(), Map.of(), "", metadata, 0, false, true); + + KubernetesDiscoveryClient discoveryClient = new KubernetesDiscoveryClient(client, properties, null, null, null); + List result = discoveryClient.getInstances("blue-service"); + Assertions.assertEquals(result.size(), 1); + DefaultKubernetesServiceInstance serviceInstance = (DefaultKubernetesServiceInstance) result.get(0); + Assertions.assertEquals(serviceInstance.getServiceId(), "blue-service"); + Assertions.assertEquals(serviceInstance.getHost(), "127.0.0.1"); + Assertions.assertEquals(serviceInstance.getPort(), 8080); + Assertions.assertFalse(serviceInstance.isSecure()); + Assertions.assertEquals(serviceInstance.getUri().toASCIIString(), "http://127.0.0.1:8080"); + Assertions.assertEquals(serviceInstance.getMetadata(), Map.of("k8s_namespace", "a", "type", "ClusterIP")); + Assertions.assertEquals(serviceInstance.podMetadata().get("labels"), Map.of("a", "b")); + Assertions.assertEquals(serviceInstance.podMetadata().get("annotations"), Map.of("c", "d")); + } + private void createEndpoints(String namespace, String name, Map labels) { client.endpoints().inNamespace(namespace) .resource(new EndpointsBuilder() diff --git a/spring-cloud-kubernetes-fabric8-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClientUtilsPodMetadataTests.java b/spring-cloud-kubernetes-fabric8-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClientUtilsPodMetadataTests.java new file mode 100644 index 00000000..d1a83f0f --- /dev/null +++ b/spring-cloud-kubernetes-fabric8-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClientUtilsPodMetadataTests.java @@ -0,0 +1,213 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.fabric8.discovery; + +import java.util.Map; +import java.util.Set; + +import io.fabric8.kubernetes.api.model.EndpointAddress; +import io.fabric8.kubernetes.api.model.EndpointAddressBuilder; +import io.fabric8.kubernetes.api.model.ObjectReferenceBuilder; +import io.fabric8.kubernetes.api.model.PodBuilder; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import org.springframework.boot.test.system.CapturedOutput; +import org.springframework.boot.test.system.OutputCaptureExtension; +import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties; + +import static org.springframework.cloud.kubernetes.fabric8.discovery.KubernetesDiscoveryClientUtils.podMetadata; + +/** + * @author wind57 + */ +@ExtendWith(OutputCaptureExtension.class) +@EnableKubernetesMockClient(https = false, crud = true) +class KubernetesDiscoveryClientUtilsPodMetadataTests { + + private KubernetesClient client; + + @AfterEach + void afterEach() { + client.pods().inAnyNamespace().delete(); + } + + /** + * service is of type ExternalName, thus no podMetadata is added. + */ + @Test + void testExternalName() { + Map serviceMetadata = Map.of("type", "ExternalName"); + boolean addPodLabels = true; + boolean addPodAnnotations = true; + KubernetesDiscoveryProperties.Metadata metadata = new KubernetesDiscoveryProperties.Metadata(false, "", false, + "", false, "", addPodLabels, addPodAnnotations); + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60L, + false, "", Set.of(), Map.of(), "", metadata, 0, false); + EndpointAddress endpointAddress = new EndpointAddressBuilder().build(); + String namespace = "default"; + + Map> result = podMetadata(client, serviceMetadata, properties, endpointAddress, + namespace); + Assertions.assertEquals(result, Map.of()); + } + + /** + * service is not of type ExternalName, but neither podLabels nor podAnnotations are + * requested. As such, podMetadata is empty. + */ + @Test + void testNotExternalName() { + Map serviceMetadata = Map.of("type", "ClusterIP"); + boolean addPodLabels = false; + boolean addPodAnnotations = false; + KubernetesDiscoveryProperties.Metadata metadata = new KubernetesDiscoveryProperties.Metadata(false, "", false, + "", false, "", addPodLabels, addPodAnnotations); + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60L, + false, "", Set.of(), Map.of(), "", metadata, 0, false); + EndpointAddress endpointAddress = new EndpointAddressBuilder().build(); + String namespace = "default"; + + Map> result = podMetadata(client, serviceMetadata, properties, endpointAddress, + namespace); + Assertions.assertEquals(result, Map.of()); + } + + /** + * service is not of type ExternalName, and podLabels are requested, but a pod does + * not exist. As such pod metadata is empty. + */ + @Test + void testNotExternalPodNotPresent() { + Map serviceMetadata = Map.of("type", "ClusterIP"); + boolean addPodLabels = true; + boolean addPodAnnotations = false; + String podName = "my-pod"; + KubernetesDiscoveryProperties.Metadata metadata = new KubernetesDiscoveryProperties.Metadata(false, "", false, + "", false, "", addPodLabels, addPodAnnotations); + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60L, + false, "", Set.of(), Map.of(), "", metadata, 0, false); + EndpointAddress endpointAddress = new EndpointAddressBuilder() + .withTargetRef(new ObjectReferenceBuilder().withKind("Pod").withName(podName).build()).build(); + String namespace = "default"; + + Map> result = podMetadata(client, serviceMetadata, properties, endpointAddress, + namespace); + Assertions.assertEquals(result, Map.of()); + } + + /** + * service is not of type ExternalName, and podLabels are requested. As such, + * podMetadata contains only pod labels. + */ + @Test + void testNotExternalNamePodLabelsRequested(CapturedOutput output) { + Map serviceMetadata = Map.of("type", "ClusterIP"); + boolean addPodLabels = true; + boolean addPodAnnotations = false; + String podName = "my-pod"; + KubernetesDiscoveryProperties.Metadata metadata = new KubernetesDiscoveryProperties.Metadata(false, "", false, + "", false, "", addPodLabels, addPodAnnotations); + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60L, + false, "", Set.of(), Map.of(), "", metadata, 0, false); + EndpointAddress endpointAddress = new EndpointAddressBuilder() + .withTargetRef(new ObjectReferenceBuilder().withKind("Pod").withName(podName).build()).build(); + String namespace = "default"; + + client.pods().inNamespace(namespace) + .resource(new PodBuilder().withNewMetadata().withName(podName) + .withLabels(Map.of("label-key", "label-value")) + .withAnnotations(Map.of("annotation-key", "annotation-value")).and().build()) + .create(); + + Map> result = podMetadata(client, serviceMetadata, properties, endpointAddress, + namespace); + Assertions.assertEquals(result.get("labels"), Map.of("label-key", "label-value")); + Assertions.assertNull(result.get("annotations")); + Assertions.assertTrue( + output.getOut().contains("adding podMetadata : {labels={label-key=label-value}} from pod : my-pod")); + } + + /** + * service is not of type ExternalName, and podAnnotations are requested. As such, + * podMetadata contains only pod annotations. + */ + @Test + void testNotExternalNamePodAnnotationsRequested(CapturedOutput output) { + Map serviceMetadata = Map.of("type", "ClusterIP"); + boolean addPodLabels = false; + boolean addPodAnnotations = true; + String podName = "my-pod"; + KubernetesDiscoveryProperties.Metadata metadata = new KubernetesDiscoveryProperties.Metadata(false, "", false, + "", false, "", addPodLabels, addPodAnnotations); + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60L, + false, "", Set.of(), Map.of(), "", metadata, 0, false); + EndpointAddress endpointAddress = new EndpointAddressBuilder() + .withTargetRef(new ObjectReferenceBuilder().withKind("Pod").withName(podName).build()).build(); + String namespace = "default"; + + client.pods().inNamespace(namespace) + .resource(new PodBuilder().withNewMetadata().withName(podName) + .withLabels(Map.of("label-key", "label-value")) + .withAnnotations(Map.of("annotation-key", "annotation-value")).and().build()) + .create(); + + Map> result = podMetadata(client, serviceMetadata, properties, endpointAddress, + namespace); + Assertions.assertNull(result.get("labels")); + Assertions.assertEquals(result.get("annotations"), Map.of("annotation-key", "annotation-value")); + Assertions.assertTrue(output.getOut() + .contains("adding podMetadata : {annotations={annotation-key=annotation-value}} from pod : my-pod")); + } + + /** + * service is not of type ExternalName, both podLabels and podAnnotations are + * requested. As such, podMetadata contains both. + */ + @Test + void testNotExternalNamePodLabelsAndAnnotationsRequested(CapturedOutput output) { + Map serviceMetadata = Map.of("type", "ClusterIP"); + boolean addPodLabels = true; + boolean addPodAnnotations = true; + String podName = "my-pod"; + KubernetesDiscoveryProperties.Metadata metadata = new KubernetesDiscoveryProperties.Metadata(false, "", false, + "", false, "", addPodLabels, addPodAnnotations); + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60L, + false, "", Set.of(), Map.of(), "", metadata, 0, false); + EndpointAddress endpointAddress = new EndpointAddressBuilder() + .withTargetRef(new ObjectReferenceBuilder().withKind("Pod").withName(podName).build()).build(); + String namespace = "default"; + + client.pods().inNamespace(namespace) + .resource(new PodBuilder().withNewMetadata().withName(podName) + .withLabels(Map.of("label-key", "label-value")) + .withAnnotations(Map.of("annotation-key", "annotation-value")).and().build()) + .create(); + + Map> result = podMetadata(client, serviceMetadata, properties, endpointAddress, + namespace); + Assertions.assertEquals(result.get("labels"), Map.of("label-key", "label-value")); + Assertions.assertEquals(result.get("annotations"), Map.of("annotation-key", "annotation-value")); + Assertions.assertTrue(output.getOut().contains( + "adding podMetadata : {annotations={annotation-key=annotation-value}, labels={label-key=label-value}} from pod : my-pod")); + } + +} diff --git a/spring-cloud-kubernetes-fabric8-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClientUtilsTests.java b/spring-cloud-kubernetes-fabric8-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClientUtilsTests.java index db5f9c79..c0a002f6 100644 --- a/spring-cloud-kubernetes-fabric8-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClientUtilsTests.java +++ b/spring-cloud-kubernetes-fabric8-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesDiscoveryClientUtilsTests.java @@ -695,14 +695,14 @@ class KubernetesDiscoveryClientUtilsTests { @Test void testServiceInstance() { KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60L, - false, "", Set.of(), Map.of(), "", null, 0, false, false); + false, "", Set.of(), Map.of(), "", KubernetesDiscoveryProperties.Metadata.DEFAULT, 0, false, false); ServicePortSecureResolver resolver = new ServicePortSecureResolver(properties); Service service = new ServiceBuilder().withMetadata(new ObjectMeta()).build(); EndpointAddress address = new EndpointAddressBuilder().withNewTargetRef().withUid("123").endTargetRef() .withIp("127.0.0.1").build(); ServiceInstance serviceInstance = KubernetesDiscoveryClientUtils.serviceInstance(resolver, service, address, - 8080, "my-service", Map.of("a", "b"), "k8s"); + 8080, "my-service", Map.of("a", "b"), "k8s", properties, null); Assertions.assertTrue(serviceInstance instanceof DefaultKubernetesServiceInstance); DefaultKubernetesServiceInstance defaultInstance = (DefaultKubernetesServiceInstance) serviceInstance; Assertions.assertEquals(defaultInstance.getInstanceId(), "123"); @@ -724,7 +724,7 @@ class KubernetesDiscoveryClientUtilsTests { .withMetadata(new ObjectMetaBuilder().withUid("123").build()).build(); ServiceInstance serviceInstance = KubernetesDiscoveryClientUtils.serviceInstance(null, service, null, -1, - "my-service", Map.of("a", "b"), "k8s"); + "my-service", Map.of("a", "b"), "k8s", KubernetesDiscoveryProperties.DEFAULT, null); Assertions.assertTrue(serviceInstance instanceof DefaultKubernetesServiceInstance); DefaultKubernetesServiceInstance defaultInstance = (DefaultKubernetesServiceInstance) serviceInstance; Assertions.assertEquals(defaultInstance.getInstanceId(), "123"); diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/configmap/Fabric8DiscoveryPodMetadataIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/configmap/Fabric8DiscoveryPodMetadataIT.java new file mode 100644 index 00000000..fc0c81d8 --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-fabric8-client-discovery/src/test/java/org/springframework/cloud/kubernetes/fabric8/configmap/Fabric8DiscoveryPodMetadataIT.java @@ -0,0 +1,166 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.fabric8.configmap; + +import java.io.InputStream; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import io.fabric8.kubernetes.api.model.EnvVar; +import io.fabric8.kubernetes.api.model.EnvVarBuilder; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.apps.Deployment; +import io.fabric8.kubernetes.api.model.networking.v1.Ingress; +import io.fabric8.kubernetes.client.KubernetesClient; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.k3s.K3sContainer; +import reactor.netty.http.client.HttpClient; +import reactor.util.retry.Retry; +import reactor.util.retry.RetryBackoffSpec; + +import org.springframework.cloud.kubernetes.commons.discovery.DefaultKubernetesServiceInstance; +import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; +import org.springframework.cloud.kubernetes.integration.tests.commons.Phase; +import org.springframework.cloud.kubernetes.integration.tests.commons.fabric8_client.Util; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpMethod; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; +import org.springframework.web.reactive.function.client.WebClient; + +/** + * @author wind57 + */ +class Fabric8DiscoveryPodMetadataIT { + + private static final String NAMESPACE = "default"; + + private static final String IMAGE_NAME = "spring-cloud-kubernetes-fabric8-client-discovery"; + + private static KubernetesClient client; + + private static Util util; + + private static final K3sContainer K3S = Commons.container(); + + @BeforeAll + static void beforeAll() throws Exception { + K3S.start(); + Commons.validateImage(IMAGE_NAME, K3S); + Commons.loadSpringCloudKubernetesImage(IMAGE_NAME, K3S); + + util = new Util(K3S); + client = util.client(); + + util.setUp(NAMESPACE); + + manifests(Phase.CREATE); + util.busybox(NAMESPACE, Phase.CREATE); + } + + @AfterAll + static void after() throws Exception { + util.busybox(NAMESPACE, Phase.DELETE); + manifests(Phase.DELETE); + Commons.cleanUp(IMAGE_NAME, K3S); + } + + @Test + void testPodMetadata() throws Exception { + + // find both pods + String[] both = K3S.execInContainer("sh", "-c", "kubectl get pods -l app=busybox -o=name --no-headers") + .getStdout().split("\n"); + // add a label to first pod + K3S.execInContainer("sh", "-c", + "kubectl label pods " + both[0].split("/")[1] + " custom-label=custom-label-value"); + // add annotation to the second pod + K3S.execInContainer("sh", "-c", + "kubectl annotate pods " + both[1].split("/")[1] + " custom-annotation=custom-annotation-value"); + + WebClient client = builder().baseUrl("http://localhost/service-instances/busybox-service").build(); + List serviceInstances = client.method(HttpMethod.GET).retrieve() + .bodyToMono(new ParameterizedTypeReference>() { + + }).retryWhen(retrySpec()).block(); + + DefaultKubernetesServiceInstance withCustomLabel = serviceInstances.stream() + .filter(x -> x.podMetadata().getOrDefault("annotations", Map.of()).isEmpty()).toList().get(0); + Assertions.assertEquals(withCustomLabel.getServiceId(), "busybox-service"); + Assertions.assertNotNull(withCustomLabel.getInstanceId()); + Assertions.assertNotNull(withCustomLabel.getHost()); + Assertions.assertEquals(withCustomLabel.getMetadata(), + Map.of("k8s_namespace", "default", "type", "ClusterIP", "port.busybox-port", "80")); + Assertions.assertTrue(withCustomLabel.podMetadata().get("labels").entrySet().stream() + .anyMatch(x -> x.getKey().equals("custom-label") && x.getValue().equals("custom-label-value"))); + + DefaultKubernetesServiceInstance withCustomAnnotation = serviceInstances.stream() + .filter(x -> !x.podMetadata().getOrDefault("annotations", Map.of()).isEmpty()).toList().get(0); + Assertions.assertEquals(withCustomAnnotation.getServiceId(), "busybox-service"); + Assertions.assertNotNull(withCustomAnnotation.getInstanceId()); + Assertions.assertNotNull(withCustomAnnotation.getHost()); + Assertions.assertEquals(withCustomAnnotation.getMetadata(), + Map.of("k8s_namespace", "default", "type", "ClusterIP", "port.busybox-port", "80")); + Assertions.assertTrue(withCustomAnnotation.podMetadata().get("annotations").entrySet().stream().anyMatch( + x -> x.getKey().equals("custom-annotation") && x.getValue().equals("custom-annotation-value"))); + } + + private static void manifests(Phase phase) { + + InputStream deploymentStream = util.inputStream("fabric8-discovery-deployment.yaml"); + InputStream serviceStream = util.inputStream("fabric8-discovery-service.yaml"); + InputStream ingressStream = util.inputStream("fabric8-discovery-ingress.yaml"); + + Deployment deployment = client.apps().deployments().load(deploymentStream).get(); + + List existing = new ArrayList<>( + deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getEnv()); + existing.add(new EnvVarBuilder().withName("SPRING_CLOUD_KUBERNETES_DISCOVERY_METADATA_ADDPODLABELS") + .withValue("true").build()); + existing.add(new EnvVarBuilder().withName("SPRING_CLOUD_KUBERNETES_DISCOVERY_METADATA_ADDPODANNOTATIONS") + .withValue("true").build()); + existing.add( + new EnvVarBuilder().withName("LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_FABRIC8_DISCOVERY") + .withValue("DEBUG").build()); + deployment.getSpec().getTemplate().getSpec().getContainers().get(0).setEnv(existing); + + Service service = client.services().load(serviceStream).get(); + Ingress ingress = client.network().v1().ingresses().load(ingressStream).get(); + + if (phase.equals(Phase.CREATE)) { + util.createAndWait(NAMESPACE, null, deployment, service, ingress, true); + } + else { + util.deleteAndWait(NAMESPACE, deployment, service, ingress); + } + + } + + private WebClient.Builder builder() { + return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create())); + } + + private RetryBackoffSpec retrySpec() { + return Retry.fixedDelay(15, Duration.ofSeconds(1)).filter(Objects::nonNull); + } + +}