From 30584aecff3ade2543f6d7c998a4badb9de41bf0 Mon Sep 17 00:00:00 2001 From: buildmaster Date: Tue, 19 Dec 2023 21:33:48 +0000 Subject: [PATCH 1/2] Bumping versions --- README.adoc | 233 ++++++++++++------ ...kingDiscoveryHealthPublishedEventTest.java | 7 +- ...tiveDiscoveryHealthPublishedEventTest.java | 9 +- .../tests/discovery/TestsDiscovery.java | 5 +- 4 files changed, 172 insertions(+), 82 deletions(-) diff --git a/README.adoc b/README.adoc index b715c097..776b7118 100644 --- a/README.adoc +++ b/README.adoc @@ -112,9 +112,36 @@ This client lets you query Kubernetes endpoints (see https://kubernetes.io/docs/ A service is typically exposed by the Kubernetes API server as a collection of endpoints that represent `http` and `https` addresses and that a client can access from a Spring Boot application running as a pod. -DiscoveryClient can also find services of type `ExternalName` (see https://kubernetes.io/docs/concepts/services-networking/service/#externalname[ExternalName services]). At the moment, external name support type of services is only available if the following property `spring.cloud.kubernetes.discovery.include-external-name-services` is set to `true` and only in the `fabric8` implementation. In a later release, support will be added for the kubernetes native client also. +DiscoveryClient can also find services of type `ExternalName` (see https://kubernetes.io/docs/concepts/services-networking/service/#externalname[ExternalName services]). At the moment, external name support type of services is only available if the following property `spring.cloud.kubernetes.discovery.include-external-name-services` is set to `true` (it is `false` by default). -This is something that you get for free by adding the following dependency inside your project: +There are 3 types of discovery clients that we support: + +1. +==== +Fabric8 Kubernetes Client +[source,xml] +---- + + org.springframework.cloud + spring-cloud-starter-kubernetes-fabric8 + +---- +==== + +2. + +==== +Kubernetes Java Client +[source,xml] +---- + + org.springframework.cloud + spring-cloud-starter-kubernetes-client + +---- +==== + +3. ==== HTTP Based `DiscoveryClient` @@ -130,28 +157,6 @@ HTTP Based `DiscoveryClient` NOTE: `spring-cloud-starter-kubernetes-discoveryclient` is designed to be used with the <>. -==== -Fabric8 Kubernetes Client -[source,xml] ----- - - org.springframework.cloud - spring-cloud-starter-kubernetes-fabric8 - ----- -==== - -==== -Kubernetes Java Client -[source,xml] ----- - - org.springframework.cloud - spring-cloud-starter-kubernetes-client - ----- -==== - To enable loading of the `DiscoveryClient`, add `@EnableDiscoveryClient` to the according configuration or application class, as the following example shows: ==== @@ -177,7 +182,19 @@ private DiscoveryClient discoveryClient; ---- ==== -You can choose to enable `DiscoveryClient` from all namespaces by setting the following property in `application.properties`: +The first question you should ask yourself is _where_ a `DiscoveryClient` supposed to discover services. In the kubernetes world, this means what namespace(s). There are 3 options here: + + - `selective namespaces`. For example: + +[source] +---- +spring.cloud.kubernetes.discovery.namespaces[0]=ns1 +spring.cloud.kubernetes.discovery.namespaces[1]=ns2 +---- + +Such a configuration makes discovery client only search for services in two namespaces `ns1` and `ns2`. + + - `all-namespaces`. ==== [source] @@ -186,15 +203,108 @@ spring.cloud.kubernetes.discovery.all-namespaces=true ---- ==== -To discover services and endpoints only from specified namespaces you should set property `all-namespaces` to `false` and set the following property in `application.properties` (in this example namespaces are: `ns1` and `ns2`). +While such an option exists, this can be a burden on both kube-api and your application. It is rare to need such a setting. + + - `one namespace`. This is the default setting, if you do not specify any of the above. It works on the rules outlined in xref:property-source-config.adoc#namespace-resolution[Namespace Resolution]. + ==== +NOTE: The above options work exactly as written for fabric8 and k8s clients. For the HTTP based client, you need to enable those options on the _server_. That can be achieved by setting them in `deployment.yaml` used to deploy the image in the cluster, using env variable(s). +==== + +For example: + [source] ---- -spring.cloud.kubernetes.discovery.namespaces[0]=ns1 -spring.cloud.kubernetes.discovery.namespaces[1]=ns2 + containers: + - name: discovery-server + image: springcloud/spring-cloud-kubernetes-discoveryserver:3.0.5-SNAPSHOT + env: + - name: SPRING_CLOUD_KUBERNETES_DISCOVERY_NAMESPACES_0 + value: "namespace-a" ---- -==== + +Once namespaces have been configured, the next question to answer is what services to discover. Think about it as what filter to apply. By default, no filtering is applied at all and all services are discovered. If you need to narrow what discovery client can find, you have two options: + +- Only take services that match certain service labels. This property is specified with: `spring.cloud.kubernetes.discovery.service-labels`. It accepts a `Map` and only those services that have such labels (as seen in `metadata.labels` in the service definition) will be taken into account. + +- The other option is to use https://docs.spring.io/spring-framework/reference/core/expressions.html[SpEL expression]. This is denoted by the `spring.cloud.kubernetes.discovery.filter` property, and its value depends on the client that you chose. If you use the fabric8 client, this SpEL expression must be created against `io.fabric8.kubernetes.api.model.Service` class. One such example could be: + +[source] +---- +spring.cloud.kubernetes.discovery.filter='#root.metadata.namespace matches "^.+A$"' +---- + +which tells discovery client to only get services that have the `metadata.namespace` that ends in upper case `A`. + +If your discovery client is based on k8s-native client, then the SpEL expression must be based on `io.kubernetes.client.openapi.models.V1Service` class. The same filter showed above would work here. + +If your discovery client is the http based one, then the SeEL expression has to be based on the same `io.kubernetes.client.openapi.models.V1Service` class, with the only distinction that this needs to be set as an env variable in the deployment yaml: + + +---- + containers: + - name: discovery-server + image: springcloud/spring-cloud-kubernetes-discoveryserver:3.0.5-SNAPSHOT + env: + - name: SPRING_CLOUD_KUBERNETES_DISCOVERY_FILTER + value: '#root.metadata.namespace matches "^.+A$"' +---- + +It's now time to think what discovery client is supposed to return back. In general, there are two methods that `DiscoveryClient` has: `getServices` and `getInstances`. + +`getServices` will return the service _names_ as seen in the `metadata.name`. + + +NOTE: This method will return unique service names, even if there are duplicates across different namespaces (that you chose for the search). + + +`getInstances` returns a `List`. Besides the usual fields that a `ServiceInstance` has, we also add some data, like namespace or pod metadata (more explanation about these will follow in the document). Here is the data that we return at the moment: + +. `instanceId` - unique id of the service instance +. `serviceId` - the name of the service (it is the same as the one reported by calling `getServices`) +. `host` - IP of the instance (or name in case of the `ExternalName` type of service) +. `port` - port number of the instance. This requires a bit more explanation, as choosing the port number has its rules: + + .. service has no port defined, 0 (zero) will be returned. + .. service has a single port defined, that one will be returned. + .. If the service has a label `primary-port-name`, we will use the port number that has the name specified in the label's value. + .. If the above label is not present, then we will use the port name specified in `spring.cloud.kubernetes.discovery.primary-port-name` to find the port number. + .. If neither of the above are specified, we will use the port named `https` or `http` to compute the port number. + .. As a last resort we wil pick the first port in the list of ports. This last option may result in non-deterministic behaviour. + +. `uri` of the service instance + +. `scheme` either `http` or `https` (depending on the `secure` result) + +. `metadata` of the service: + +.. `labels` (if requested via `spring.cloud.kubernetes.discovery.metadata.add-labels=true`). Label keys can be "prefixed" with the value of `spring.cloud.kubernetes.discovery.metadata.labels-prefix` if it is set. + +.. `annotations` (if requested via `spring.cloud.kubernetes.discovery.metadata.add-annotations=true`). Annotations keys can be "prefixed" with the value of `spring.cloud.kubernetes.discovery.metadata.annotations-prefix` if it is set. + +.. `ports` (if requested via `spring.cloud.kubernetes.discovery.metadata.add-ports=true`). Port keys can be "prefixed" with the value of `spring.cloud.kubernetes.discovery.metadata.ports-prefix` if it is set. + +.. `k8s_namespace` with the value of the namespace where instance resides. + +.. `type` that holds the service type, for example `ClusterIP` or `ExternalName` + +. `secure` if the port that was discovered should be treated as secure. We will use the same rules outlined above to find the port name and number, and then: + +.. If this service has a label called `secured` with any of the values : `["true", "on", "yes", "1"]`, then treat the port that was found as secure. + +.. If such a label is not found, search for an annotation called `secured` and apply the same above rules. + +.. If this port number is part of `spring.cloud.kubernetes.discovery.known-secure-ports` (by default this value holds `[443, 8443]`), treat port number as secured. + +.. Last resort is to see if port name matches `https`; if it does treat this port as secured. + +. `namespace` - the namespace of the found instance. + +. `pod-metadata` labels and annotations of the service instance (pod), in the form of `Map>`. This support needs to be enabled via `spring.cloud.kubernetes.discovery.metadata.add-pod-labels=true` and/or `spring.cloud.kubernetes.discovery.metadata.add-pod-annotaations=true` + +''' + To discover service endpoint addresses that are not marked as "ready" by the kubernetes api server, you can set the following property in `application.properties` (default: false): @@ -204,42 +314,27 @@ To discover service endpoint addresses that are not marked as "ready" by the kub spring.cloud.kubernetes.discovery.include-not-ready-addresses=true ---- NOTE: This might be useful when discovering services for monitoring purposes, and would enable inspecting the `/health` endpoint of not-ready service instances. -==== -If your service exposes multiple ports, you will need to specify which port the `DiscoveryClient` should use. -The `DiscoveryClient` will choose the port using the following logic. -1. If the service has a label `primary-port-name` it will use the port with the name specified in the label's value. -2. If no label is present, then the port name specified in `spring.cloud.kubernetes.discovery.primary-port-name` will be used. -3. If neither of the above are specified it will use the port named `https`. -4. If none of the above conditions are met it will use the port named `http`. -5. As a last resort it wil pick the first port in the list of ports. - -WARNING: The last option may result in non-deterministic behaviour. -Please make sure to configure your service and/or application accordingly. - -By default all of the ports and their names will be added to the metadata of the `ServiceInstance`. - -As said before, if you want to get the list of `ServiceInstance` to also include the `ExternalName` type services, you need to enable that support via: `spring.cloud.kubernetes.discovery.include-external-name-services=true`. As such, when calling `DiscoveryClient::getInstances` those will be returned also. You can distinguish between `ExternalName` and any other types by inspecting `ServiceInstance::getMetadata` and lookup for a field called `type`. This will be the type of the service returned : `ExternalName`/`ClusterIP`, etc. - -`ServiceInstance` can include the labels and annotations of specific pods from the underlying service instance. To obtain such information, you need to also enable: - -`spring.cloud.kubernetes.discovery.metadata.add-pod-labels=true` and/or `spring.cloud.kubernetes.discovery.metadata.add-pod-annotations=true`. At the moment, such functionality is present only in the fabric8 client implementation, but will be added to the kubernetes native client in a later release. +If you want to get the list of `ServiceInstance` to also include the `ExternalName` type services, you need to enable that support via: `spring.cloud.kubernetes.discovery.include-external-name-services=true`. As such, when calling `DiscoveryClient::getInstances` those will be returned also. You can distinguish between `ExternalName` and any other types by inspecting `ServiceInstance::getMetadata` and lookup for a field called `type`. This will be the type of the service returned : `ExternalName`/`ClusterIP`, etc. If, for any reason, you need to disable the `DiscoveryClient`, you can set the following property in `application.properties`: ==== [source] ---- -spring.cloud.kubernetes.discovery.enabled=false +spring.main.cloud-platform=NONE ---- -==== + +Note that the support of discovery client is _automatic_, depending on where you run the application. So the above setting might not be needed. Some Spring Cloud components use the `DiscoveryClient` in order to obtain information about the local service instance. For this to work, you need to align the Kubernetes service name with the `spring.application.name` property. NOTE: `spring.application.name` has no effect as far as the name registered for the application within Kubernetes +''' + Spring Cloud Kubernetes can also watch the Kubernetes service catalog for changes and update the `DiscoveryClient` implementation accordingly. By "watch" we mean that we will publish a heartbeat event every `spring.cloud.kubernetes.discovery.catalog-services-watch-delay` milliseconds (by default it is `30000`). The heartbeat event will contain the target references (and their namespaces of the addresses of all endpoints @@ -251,7 +346,8 @@ The endpoints will be queried in either : - specific namespaces (enabled via `spring.cloud.kubernetes.discovery.namespaces`), for example: -``` +[source] +---- spring: cloud: kubernetes: @@ -259,7 +355,7 @@ spring: namespaces: - namespace-a - namespace-b -``` +---- - we will use: xref:property-source-config.adoc#namespace-resolution[Namespace Resolution] if the above two paths are not taken. @@ -268,7 +364,8 @@ In order to enable this functionality you need to add By default, we use the `Endpoints`(see https://kubernetes.io/docs/concepts/services-networking/service/#endpoints) API to find out the current state of services. There is another way though, via `EndpointSlices` (https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/). Such support can be enabled via a property: `spring.cloud.kubernetes.discovery.use-endpoint-slices=true` (by default it is `false`). Of course, your cluster has to support it also. As a matter of fact, if you enable this property, but your cluster does not support it, we will fail starting the application. If you decide to enable such support, you also need proper Role/ClusterRole set-up. For example: -``` +[source] +---- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -278,7 +375,7 @@ rules: - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] verbs: ["get", "list", "watch"] -``` +---- == Kubernetes native service discovery @@ -320,9 +417,11 @@ observed `ConfigMap` instances. Everything that follows is explained mainly referring to examples using ConfigMaps, but the same stands for Secrets, i.e.: every feature is supported for both. -The default behavior is to create a `Fabric8ConfigMapPropertySource` (or a `KubernetesClientConfigMapPropertySource`) based on a Kubernetes `ConfigMap` that has a `metadata.name` value of either the name of -your Spring application (as defined by its `spring.application.name` property) or a custom name defined within the -`application.properties` file under the following key: `spring.cloud.kubernetes.config.name`. +The default behavior is to create a `Fabric8ConfigMapPropertySource` (or a `KubernetesClientConfigMapPropertySource`) based on a Kubernetes `ConfigMap` that has `metadata.name` of either: + + - value of `spring.cloud.kubernetes.config.name` + - value of your Spring application (as defined by `spring.application.name` property) + - the String literal `"application"` However, more advanced configuration is possible where you can use multiple `ConfigMap` instances. The `spring.cloud.kubernetes.config.sources` list makes this possible. @@ -380,16 +479,19 @@ data: my-app-k8s.yaml: |- .. my-app-dev.yaml: |- + .. + not-my-app.yaml: |- .. someProp: someValue ---- ==== -These is what we will end-up loading: +This is what we will end-up loading: - `my-app.yaml` treated as a file - `my-app-k8s.yaml` treated as a file - `my-app-dev.yaml` _ignored_, since `dev` is _not_ an active profile + - `not-my-app.yaml` _ignored_, since it does not match `spring.application.name` - `someProp: someValue` plain property The single exception to the aforementioned flow is when the `ConfigMap` contains a *single* key that indicates @@ -1366,23 +1468,10 @@ will only take effect when set in `bootstrap.{properties|yml}` when you have `sp === Breaking Changes In 3.0.x - - In versions of Spring Cloud Kubernetes prior to `3.0.x`, Kubernetes awareness was implemented using `spring.cloud.kubernetes.enabled` property. This +In versions of Spring Cloud Kubernetes prior to `3.0.x`, Kubernetes awareness was implemented using `spring.cloud.kubernetes.enabled` property. This property was removed and is un-supported. Instead, we use Spring Boot API: https://docs.spring.io/spring-boot/docs/current/api/org/springframework/boot/autoconfigure/condition/ConditionalOnCloudPlatform.html[ConditionalOnCloudPlatform]. If it is needed to explicitly enable or disable this awareness, use `spring.main.cloud-platform=NONE/KUBERNETES`. - - Another breaking change is the additional `list` verb needed for loading configmaps/secrets. For example: - -``` -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cluster-role -rules: - - apiGroups: ["", "extensions", "apps", "discovery.k8s.io"] - resources: ["configmaps", "pods", "services", "endpoints", "secrets", "endpointslices"] - verbs: ["get", "list", "watch"] -``` - === Kubernetes Profile Autoconfiguration When the application runs as a pod inside Kubernetes, a Spring profile named `kubernetes` automatically gets activated. diff --git a/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/BlockingDiscoveryHealthPublishedEventTest.java b/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/BlockingDiscoveryHealthPublishedEventTest.java index eb2e15aa..416f480b 100644 --- a/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/BlockingDiscoveryHealthPublishedEventTest.java +++ b/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/BlockingDiscoveryHealthPublishedEventTest.java @@ -27,9 +27,10 @@ import org.springframework.boot.test.context.SpringBootTest; * * @author wind57 */ -@SpringBootTest(properties = { "spring.main.cloud-platform=kubernetes", "spring.cloud.config.enabled=false", - "spring.cloud.kubernetes.discovery.discovery-server-url=http://example", - "spring.cloud.discovery.reactive.enabled=false" }, +@SpringBootTest( + properties = { "spring.main.cloud-platform=kubernetes", "spring.cloud.config.enabled=false", + "spring.cloud.kubernetes.discovery.discovery-server-url=http://example", + "spring.cloud.discovery.reactive.enabled=false" }, classes = { HealthEventListenerConfiguration.class, App.class }) class BlockingDiscoveryHealthPublishedEventTest { diff --git a/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/ReactiveDiscoveryHealthPublishedEventTest.java b/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/ReactiveDiscoveryHealthPublishedEventTest.java index 87a37ed2..72ca89a8 100644 --- a/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/ReactiveDiscoveryHealthPublishedEventTest.java +++ b/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/ReactiveDiscoveryHealthPublishedEventTest.java @@ -29,10 +29,11 @@ import org.springframework.boot.test.context.SpringBootTest; * * @author wind57 */ -@SpringBootTest(properties = { "spring.main.cloud-platform=kubernetes", "spring.cloud.config.enabled=false", - "spring.cloud.kubernetes.discovery.discovery-server-url=http://example", - // disable blocking implementation - "spring.cloud.discovery.blocking.enabled=false" }, +@SpringBootTest( + properties = { "spring.main.cloud-platform=kubernetes", "spring.cloud.config.enabled=false", + "spring.cloud.kubernetes.discovery.discovery-server-url=http://example", + // disable blocking implementation + "spring.cloud.discovery.blocking.enabled=false" }, classes = { HealthEventListenerConfiguration.class, App.class }) class ReactiveDiscoveryHealthPublishedEventTest { diff --git a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/tests/discovery/TestsDiscovery.java b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/tests/discovery/TestsDiscovery.java index 38d86ea7..f3605769 100644 --- a/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/tests/discovery/TestsDiscovery.java +++ b/spring-cloud-kubernetes-test-support/src/main/java/org/springframework/cloud/kubernetes/tests/discovery/TestsDiscovery.java @@ -63,9 +63,8 @@ public class TestsDiscovery { Launcher launcher = session.getLauncher(); TestPlan testPlan = launcher.discover(request); testPlan.getRoots().stream().flatMap(x -> testPlan.getChildren(x).stream()) - .map(TestIdentifier::getLegacyReportingName).sorted().forEach(test -> - System.out.println("spring.cloud.k8s.test.to.run -> " + test) - ); + .map(TestIdentifier::getLegacyReportingName).sorted() + .forEach(test -> System.out.println("spring.cloud.k8s.test.to.run -> " + test)); } } From bf5b1fd4b030934d1de857fa809b8204282b980c Mon Sep 17 00:00:00 2001 From: erabii Date: Thu, 21 Dec 2023 21:24:56 +0200 Subject: [PATCH 2/2] fix 977 (#1540) --- docs/src/main/asciidoc/discovery-client.adoc | 63 ++++-- .../catalog/KubernetesCatalogWatch.java | 3 +- ...lOnHttpDiscoveryCatalogWatcherEnabled.java | 41 ++++ .../KubernetesDiscoveryConstants.java | 16 ++ .../DiscoveryCatalogWatcherController.java | 48 +++++ .../DiscoveryServerApplication.java | 2 + .../discoveryserver/HeartBeatListener.java | 73 +++++++ ...iscoveryCatalogWatcherControllerTests.java | 47 +++++ ...iscoveryServerApplicationContextTests.java | 141 +++++++++++++ .../discoveryserver/HeartbeatTest.java | 99 +++++++++ .../discovery/ConfigServerBootstrapper.java | 2 +- .../discovery/KubernetesCatalogWatch.java | 82 ++++++++ ...bernetesCatalogWatchAutoConfiguration.java | 77 +++++++ ...coveryClientBlockingAutoConfiguration.java | 9 +- .../KubernetesReactiveDiscoveryClient.java | 4 +- ...ot.autoconfigure.AutoConfiguration.imports | 1 + ...tesCatalogWatchAutoConfigurationTests.java | 133 +++++++++++++ .../KubernetesCatalogWatchTests.java | 188 ++++++++++++++++++ ...rnetesDiscoveryAutoConfigurationTests.java | 11 +- .../discovery/KubernetesCatalogWatch.java | 3 +- .../discoveryclient/it/HeartbeatListener.java | 49 +++++ ...ubernetesDiscoveryClientApplicationIt.java | 19 +- ...iscoveryClientFilterNamespaceDelegate.java | 47 ++++- .../discoveryclient/it/DiscoveryClientIT.java | 62 +++++- ...ernetes-discoveryclient-it-deployment.yaml | 3 + ...kubernetes-discoveryserver-deployment.yaml | 5 + 26 files changed, 1187 insertions(+), 41 deletions(-) create mode 100644 spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/ConditionalOnHttpDiscoveryCatalogWatcherEnabled.java create mode 100644 spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/main/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryCatalogWatcherController.java create mode 100644 spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/main/java/org/springframework/cloud/kubernetes/discoveryserver/HeartBeatListener.java create mode 100644 spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/test/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryCatalogWatcherControllerTests.java create mode 100644 spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/test/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryServerApplicationContextTests.java create mode 100644 spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/test/java/org/springframework/cloud/kubernetes/discoveryserver/HeartbeatTest.java create mode 100644 spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatch.java create mode 100644 spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatchAutoConfiguration.java create mode 100644 spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatchAutoConfigurationTests.java create mode 100644 spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatchTests.java create mode 100644 spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/main/java/org/springframework/cloud/kubernetes/discoveryclient/it/HeartbeatListener.java diff --git a/docs/src/main/asciidoc/discovery-client.adoc b/docs/src/main/asciidoc/discovery-client.adoc index 00d893f3..f2b439d8 100644 --- a/docs/src/main/asciidoc/discovery-client.adoc +++ b/docs/src/main/asciidoc/discovery-client.adoc @@ -229,32 +229,65 @@ NOTE: `spring.application.name` has no effect as far as the name registered for ''' -Spring Cloud Kubernetes can also watch the Kubernetes service catalog for changes and update the -`DiscoveryClient` implementation accordingly. By "watch" we mean that we will publish a heartbeat event every `spring.cloud.kubernetes.discovery.catalog-services-watch-delay` -milliseconds (by default it is `30000`). The heartbeat event will contain the target references (and their namespaces of the addresses of all endpoints +Spring Cloud Kubernetes can also watch the Kubernetes service catalog for changes and update the `DiscoveryClient` implementation accordingly. In order to enable this functionality you need to add +`@EnableScheduling` on a configuration class in your application. By "watch", we mean that we will publish a heartbeat event every `spring.cloud.kubernetes.discovery.catalog-services-watch-delay` +milliseconds (by default it is `30000`). For the http discovery server this must be an environment variable set in deployment yaml: + +---- + containers: + - name: discovery-server + image: springcloud/spring-cloud-kubernetes-discoveryserver:3.0.5-SNAPSHOT + env: + - name: SPRING_CLOUD_KUBERNETES_DISCOVERY_CATALOGSERVICESWATCHDELAY + value: 3000 +---- + +The heartbeat event will contain the target references (and their namespaces of the addresses of all endpoints (for the exact details of what will get returned you can take a look inside `KubernetesCatalogWatch`). This is an implementation detail, and listeners of the heartbeat event should not rely on the details. Instead, they should see if there are differences between two subsequent heartbeats via `equals` method. We will take care to return a correct implementation that adheres to the equals contract. The endpoints will be queried in either : - - all namespaces (enabled via `spring.cloud.kubernetes.discovery.all-namespaces=true`) + - `all-namespaces` (enabled via `spring.cloud.kubernetes.discovery.all-namespaces=true`) - - specific namespaces (enabled via `spring.cloud.kubernetes.discovery.namespaces`), for example: + - `selective namespaces` (enabled via `spring.cloud.kubernetes.discovery.namespaces`), for example: + + - `one namespace` via xref:property-source-config.adoc#namespace-resolution[Namespace Resolution] if the above two paths are not taken. + +NOTE: If, for any reasons, you want to disable catalog watcher, you need to set `spring.cloud.kubernetes.discovery.catalog-services-watch.enabled=false`. For the http discovery server, this needs to be an environment variable set in deployment for example: [source] ---- -spring: - cloud: - kubernetes: - discovery: - namespaces: - - namespace-a - - namespace-b +SPRING_CLOUD_KUBERNETES_DISCOVERY_CATALOGSERVICESWATCH_ENABLED=FALSE ---- -- we will use: xref:property-source-config.adoc#namespace-resolution[Namespace Resolution] if the above two paths are not taken. +The functionality of catalog watch works for all 3 discovery clients that we support, with some caveats that you need to be aware of in case of the http client. + +- The first is that this functionality is disabled by default, and it needs to be enabled in two places: + + * in discovery server via an environment variable in the deployment manifest, for example: ++ +---- +containers: + - name: discovery-server + image: springcloud/spring-cloud-kubernetes-discoveryserver:3.0.5-SNAPSHOT + env: + - name: SPRING_CLOUD_KUBERNETES_HTTP_DISCOVERY_CATALOG_WATCHER_ENABLED + value: "TRUE" +---- ++ + +* in discovery client, via a property in your `application.properties` for example: ++ +---- +spring.cloud.kubernetes.http.discovery.catalog.watcher.enabled=true +---- ++ + +- The second point is that this is only supported since version `3.0.6` and upwards. +- Since http discovery has _two_ components : server and client, we strongly recommend to align versions between them, otherwise things might not work. +- If you decide to disable catalog watcher, you need to disable it in both server and client. + -In order to enable this functionality you need to add -`@EnableScheduling` on a configuration class in your application. By default, we use the `Endpoints`(see https://kubernetes.io/docs/concepts/services-networking/service/#endpoints) API to find out the current state of services. There is another way though, via `EndpointSlices` (https://kubernetes.io/docs/concepts/services-networking/endpoint-slices/). Such support can be enabled via a property: `spring.cloud.kubernetes.discovery.use-endpoint-slices=true` (by default it is `false`). Of course, your cluster has to support it also. As a matter of fact, if you enable this property, but your cluster does not support it, we will fail starting the application. If you decide to enable such support, you also need proper Role/ClusterRole set-up. For example: diff --git a/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatch.java b/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatch.java index 2a3d5d44..bb9fbe5c 100644 --- a/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatch.java +++ b/spring-cloud-kubernetes-client-discovery/src/main/java/org/springframework/cloud/kubernetes/client/discovery/catalog/KubernetesCatalogWatch.java @@ -36,6 +36,7 @@ import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.core.log.LogAccessor; import org.springframework.scheduling.annotation.Scheduled; +import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.CATALOG_WATCH_PROPERTY_WITH_DEFAULT_VALUE; import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.DISCOVERY_GROUP; import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.DISCOVERY_VERSION; import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.ENDPOINT_SLICE; @@ -67,7 +68,7 @@ class KubernetesCatalogWatch implements ApplicationEventPublisherAware { this.publisher = publisher; } - @Scheduled(fixedDelayString = "${spring.cloud.kubernetes.discovery.catalogServicesWatchDelay:30000}") + @Scheduled(fixedDelayString = "${" + CATALOG_WATCH_PROPERTY_WITH_DEFAULT_VALUE + "}") void catalogServicesWatch() { try { diff --git a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/ConditionalOnHttpDiscoveryCatalogWatcherEnabled.java b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/ConditionalOnHttpDiscoveryCatalogWatcherEnabled.java new file mode 100644 index 00000000..40a8392a --- /dev/null +++ b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/ConditionalOnHttpDiscoveryCatalogWatcherEnabled.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.commons.discovery; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +/** + * Provides a more succinct conditional for: + * spring.cloud.kubernetes.http.discovery.client.catalog.watcher.enabled. + * + * @author wind57 + */ +@Target({ ElementType.TYPE, ElementType.METHOD }) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Inherited +@ConditionalOnProperty(value = "spring.cloud.kubernetes.http.discovery.catalog.watcher.enabled", matchIfMissing = false) +public @interface ConditionalOnHttpDiscoveryCatalogWatcherEnabled { + +} diff --git a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryConstants.java b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryConstants.java index 6865d52a..59d599ba 100644 --- a/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryConstants.java +++ b/spring-cloud-kubernetes-commons/src/main/java/org/springframework/cloud/kubernetes/commons/discovery/KubernetesDiscoveryConstants.java @@ -82,4 +82,20 @@ public final class KubernetesDiscoveryConstants { */ public static final String SECURED = "secured"; + /** + * catalog watch delay property name. + */ + public static final String CATALOG_WATCH_PROPERTY_NAME = "spring.cloud.kubernetes.discovery.catalogServicesWatchDelay"; + + /** + * default delay for the configuration watcher. + */ + public static final String CATALOG_WATCHER_DEFAULT_DELAY = "30000"; + + /** + * catalog watch delay property name with default value. + */ + public static final String CATALOG_WATCH_PROPERTY_WITH_DEFAULT_VALUE = CATALOG_WATCH_PROPERTY_NAME + ":" + + CATALOG_WATCHER_DEFAULT_DELAY; + } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/main/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryCatalogWatcherController.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/main/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryCatalogWatcherController.java new file mode 100644 index 00000000..47410e66 --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/main/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryCatalogWatcherController.java @@ -0,0 +1,48 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.discoveryserver; + +import java.util.List; + +import reactor.core.publisher.Mono; + +import org.springframework.cloud.kubernetes.commons.discovery.ConditionalOnHttpDiscoveryCatalogWatcherEnabled; +import org.springframework.cloud.kubernetes.commons.discovery.ConditionalOnKubernetesCatalogEnabled; +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * @author wind57 + */ +@RestController +@ConditionalOnKubernetesCatalogEnabled +@ConditionalOnHttpDiscoveryCatalogWatcherEnabled +class DiscoveryCatalogWatcherController { + + private final HeartBeatListener heartBeatListener; + + DiscoveryCatalogWatcherController(HeartBeatListener heartBeatListener) { + this.heartBeatListener = heartBeatListener; + } + + @GetMapping("/state") + Mono> state() { + return Mono.defer(() -> Mono.just(heartBeatListener.lastState().get())); + } + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/main/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryServerApplication.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/main/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryServerApplication.java index b2cd0375..74881029 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/main/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryServerApplication.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/main/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryServerApplication.java @@ -18,11 +18,13 @@ package org.springframework.cloud.kubernetes.discoveryserver; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.scheduling.annotation.EnableScheduling; /** * @author Ryan Baxter */ @SpringBootApplication +@EnableScheduling public class DiscoveryServerApplication { public static void main(String[] args) { diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/main/java/org/springframework/cloud/kubernetes/discoveryserver/HeartBeatListener.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/main/java/org/springframework/cloud/kubernetes/discoveryserver/HeartBeatListener.java new file mode 100644 index 00000000..85aece0e --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/main/java/org/springframework/cloud/kubernetes/discoveryserver/HeartBeatListener.java @@ -0,0 +1,73 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.discoveryserver; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.commons.logging.LogFactory; + +import org.springframework.cloud.client.discovery.event.HeartbeatEvent; +import org.springframework.cloud.kubernetes.commons.discovery.ConditionalOnHttpDiscoveryCatalogWatcherEnabled; +import org.springframework.cloud.kubernetes.commons.discovery.ConditionalOnKubernetesCatalogEnabled; +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.context.ApplicationListener; +import org.springframework.core.env.Environment; +import org.springframework.core.log.LogAccessor; +import org.springframework.stereotype.Component; + +import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.CATALOG_WATCHER_DEFAULT_DELAY; +import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.CATALOG_WATCH_PROPERTY_NAME; + +/** + * Listener for a HeartbeatEvent that comes from KubernetesCatalogWatch. + * + * @author wind57 + */ +@Component +@ConditionalOnKubernetesCatalogEnabled +@ConditionalOnHttpDiscoveryCatalogWatcherEnabled +class HeartBeatListener implements ApplicationListener { + + private static final LogAccessor LOG = new LogAccessor(LogFactory.getLog(HeartBeatListener.class)); + + private final AtomicReference> lastState = new AtomicReference<>(List.of()); + + HeartBeatListener(Environment environment) { + String watchDelay = environment.getProperty(CATALOG_WATCH_PROPERTY_NAME); + if (watchDelay != null) { + LOG.debug("using delay : " + watchDelay); + } + else { + LOG.debug("using default watch delay : " + CATALOG_WATCHER_DEFAULT_DELAY); + } + } + + @Override + @SuppressWarnings("unchecked") + public void onApplicationEvent(HeartbeatEvent event) { + LOG.debug(() -> "received heartbeat event"); + List state = (List) event.getValue(); + LOG.debug(() -> "state received : " + state); + lastState.set(state); + } + + AtomicReference> lastState() { + return lastState; + } + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/test/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryCatalogWatcherControllerTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/test/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryCatalogWatcherControllerTests.java new file mode 100644 index 00000000..7964f8e5 --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/test/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryCatalogWatcherControllerTests.java @@ -0,0 +1,47 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.discoveryserver; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import reactor.test.StepVerifier; + +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; + +/** + * @author wind57 + */ +class DiscoveryCatalogWatcherControllerTests { + + private final HeartBeatListener heartBeatListener = Mockito.mock(HeartBeatListener.class); + + @Test + void test() { + Mockito.when(heartBeatListener.lastState()) + .thenReturn(new AtomicReference<>(List.of(new EndpointNameAndNamespace("one", "two")))); + + DiscoveryCatalogWatcherController catalogWatcherController = new DiscoveryCatalogWatcherController( + heartBeatListener); + + StepVerifier.create(catalogWatcherController.state()) + .expectNext(List.of(new EndpointNameAndNamespace("one", "two"))).verifyComplete(); + } + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/test/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryServerApplicationContextTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/test/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryServerApplicationContextTests.java new file mode 100644 index 00000000..10a9b2ac --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/test/java/org/springframework/cloud/kubernetes/discoveryserver/DiscoveryServerApplicationContextTests.java @@ -0,0 +1,141 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.discoveryserver; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.cloud.kubernetes.client.discovery.reactive.KubernetesInformerReactiveDiscoveryClient; +import org.springframework.context.annotation.Bean; + +/** + * @author wind57 + */ +class DiscoveryServerApplicationContextTests { + + @Nested + @SpringBootTest(classes = TestConfig.class, + properties = "spring.cloud.kubernetes.http.discovery.catalog.watcher.enabled=true") + class BothControllersPresent { + + @Autowired + private ObjectProvider discoveryServerController; + + @Autowired + private ObjectProvider discoveryCatalogWatcherController; + + @Autowired + private ObjectProvider heartBeatListener; + + @Test + void test() { + Assertions.assertNotNull(discoveryServerController.getIfAvailable()); + Assertions.assertNotNull(discoveryCatalogWatcherController.getIfAvailable()); + Assertions.assertNotNull(heartBeatListener.getIfAvailable()); + } + + } + + @Nested + @SpringBootTest(classes = TestConfig.class, + properties = { "spring.cloud.kubernetes.discovery.catalog-services-watch.enabled=false", + "spring.cloud.kubernetes.http.discovery.catalog.watcher.enabled=true" }) + class CatalogControllerNotPresentOne { + + @Autowired + private ObjectProvider discoveryServerController; + + @Autowired + private ObjectProvider discoveryCatalogWatcherController; + + @Autowired + private ObjectProvider heartBeatListener; + + @Test + void test() { + Assertions.assertNotNull(discoveryServerController.getIfAvailable()); + Assertions.assertNull(discoveryCatalogWatcherController.getIfAvailable()); + Assertions.assertNull(heartBeatListener.getIfAvailable()); + } + + } + + @Nested + @SpringBootTest(classes = TestConfig.class, + properties = { "spring.cloud.kubernetes.discovery.catalog-services-watch.enabled=true", + "spring.cloud.kubernetes.http.discovery.catalog.watcher.enabled=false" }) + class CatalogControllerNotPresentTwo { + + @Autowired + private ObjectProvider discoveryServerController; + + @Autowired + private ObjectProvider discoveryCatalogWatcherController; + + @Autowired + private ObjectProvider heartBeatListener; + + @Test + void test() { + Assertions.assertNotNull(discoveryServerController.getIfAvailable()); + Assertions.assertNull(discoveryCatalogWatcherController.getIfAvailable()); + Assertions.assertNull(heartBeatListener.getIfAvailable()); + } + + } + + @Nested + @SpringBootTest(classes = TestConfig.class, + properties = { "spring.cloud.kubernetes.discovery.catalog-services-watch.enabled=false", + "spring.cloud.kubernetes.http.discovery.catalog.watcher.enabled=false" }) + class CatalogControllerNotPresentThree { + + @Autowired + private ObjectProvider discoveryServerController; + + @Autowired + private ObjectProvider discoveryCatalogWatcherController; + + @Autowired + private ObjectProvider heartBeatListener; + + @Test + void test() { + Assertions.assertNotNull(discoveryServerController.getIfAvailable()); + Assertions.assertNull(discoveryCatalogWatcherController.getIfAvailable()); + Assertions.assertNull(heartBeatListener.getIfAvailable()); + } + + } + + @TestConfiguration + static class TestConfig { + + @Bean + KubernetesInformerReactiveDiscoveryClient discoveryClient() { + return Mockito.mock(KubernetesInformerReactiveDiscoveryClient.class); + } + + } + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/test/java/org/springframework/cloud/kubernetes/discoveryserver/HeartbeatTest.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/test/java/org/springframework/cloud/kubernetes/discoveryserver/HeartbeatTest.java new file mode 100644 index 00000000..9de17260 --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-discoveryserver/src/test/java/org/springframework/cloud/kubernetes/discoveryserver/HeartbeatTest.java @@ -0,0 +1,99 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.discoveryserver; + +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.reactive.AutoConfigureWebTestClient; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.cloud.client.discovery.event.HeartbeatEvent; +import org.springframework.cloud.kubernetes.client.discovery.reactive.KubernetesInformerReactiveDiscoveryClient; +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.ApplicationEventPublisherAware; +import org.springframework.context.annotation.Bean; +import org.springframework.test.web.reactive.server.WebTestClient; + +/** + * @author wind57 + */ +@SpringBootTest(classes = HeartbeatTest.TestConfig.class, + properties = "spring.cloud.kubernetes.http.discovery.catalog.watcher.enabled=true") +@AutoConfigureWebTestClient +class HeartbeatTest { + + @Autowired + private WebTestClient client; + + @Autowired + private ApplicationContext context; + + @Test + void testHeartbeat() { + Assertions.assertTrue(true); + client.get().uri("/state").exchange().expectStatus().is2xxSuccessful().expectBody().json("[]"); + + context.getBean(HeartbeatPublisher.class).publishEvent(); + client.get().uri("/state").exchange().expectStatus().is2xxSuccessful().expectBody().json(""" + [ + { + "endpointName":"endpoint-name", + "namespace":"namespaceA" + } + ] + """); + } + + @TestConfiguration + static class TestConfig { + + @Bean + KubernetesInformerReactiveDiscoveryClient client() { + return Mockito.mock(KubernetesInformerReactiveDiscoveryClient.class); + } + + @Bean + HeartbeatPublisher heartbeatPublisher() { + return new HeartbeatPublisher(); + } + + } + + static class HeartbeatPublisher implements ApplicationEventPublisherAware { + + private ApplicationEventPublisher publisher; + + @Override + public void setApplicationEventPublisher(ApplicationEventPublisher publisher) { + this.publisher = publisher; + } + + void publishEvent() { + publisher.publishEvent( + new HeartbeatEvent("test", List.of(new EndpointNameAndNamespace("endpoint-name", "namespaceA")))); + } + + } + +} diff --git a/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/ConfigServerBootstrapper.java b/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/ConfigServerBootstrapper.java index 2e96d8b6..88b9c2c2 100644 --- a/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/ConfigServerBootstrapper.java +++ b/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/ConfigServerBootstrapper.java @@ -79,7 +79,7 @@ class ConfigServerBootstrapper extends KubernetesConfigServerBootstrapper { .orElseGet(() -> KubernetesDiscoveryProperties.DEFAULT); KubernetesDiscoveryClientBlockingAutoConfiguration autoConfiguration = new KubernetesDiscoveryClientBlockingAutoConfiguration(); DiscoveryClient discoveryClient = autoConfiguration - .kubernetesDiscoveryClient(autoConfiguration.restTemplate(), kubernetesDiscoveryProperties); + .kubernetesDiscoveryClient(autoConfiguration.restTemplateBuilder(), kubernetesDiscoveryProperties); return discoveryClient::getInstances; } diff --git a/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatch.java b/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatch.java new file mode 100644 index 00000000..05a63bb1 --- /dev/null +++ b/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatch.java @@ -0,0 +1,82 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.discovery; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.commons.logging.LogFactory; + +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.cloud.client.discovery.event.HeartbeatEvent; +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.ApplicationEventPublisherAware; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.core.log.LogAccessor; +import org.springframework.http.HttpMethod; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.web.client.RestTemplate; + +import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.CATALOG_WATCH_PROPERTY_WITH_DEFAULT_VALUE; + +/** + * @author wind57 + */ +final class KubernetesCatalogWatch implements ApplicationEventPublisherAware { + + private static final ParameterizedTypeReference> TYPE = new ParameterizedTypeReference<>() { + + }; + + private static final LogAccessor LOG = new LogAccessor(LogFactory.getLog(KubernetesCatalogWatch.class)); + + private final AtomicReference> catalogState = new AtomicReference<>(List.of()); + + private final RestTemplate restTemplate; + + private ApplicationEventPublisher publisher; + + KubernetesCatalogWatch(RestTemplateBuilder builder, KubernetesDiscoveryProperties properties) { + this.restTemplate = builder.rootUri(properties.discoveryServerUrl()).build(); + } + + @Override + public void setApplicationEventPublisher(ApplicationEventPublisher publisher) { + this.publisher = publisher; + } + + @Scheduled(fixedDelayString = "${" + CATALOG_WATCH_PROPERTY_WITH_DEFAULT_VALUE + "}") + public void catalogServicesWatch() { + try { + List currentState = restTemplate.exchange("/state", HttpMethod.GET, null, TYPE) + .getBody(); + + if (!catalogState.get().equals(currentState)) { + LOG.debug(() -> "Received update from kubernetes discovery http client: " + currentState); + publisher.publishEvent(new HeartbeatEvent(this, currentState)); + } + + catalogState.set(currentState); + } + catch (Exception e) { + LOG.error(e, () -> "Error watching Kubernetes Services"); + } + } + +} diff --git a/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatchAutoConfiguration.java b/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatchAutoConfiguration.java new file mode 100644 index 00000000..de14ba3e --- /dev/null +++ b/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatchAutoConfiguration.java @@ -0,0 +1,77 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.discovery; + +import org.apache.commons.logging.LogFactory; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnCloudPlatform; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.cloud.CloudPlatform; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.cloud.client.ConditionalOnDiscoveryEnabled; +import org.springframework.cloud.kubernetes.commons.discovery.ConditionalOnHttpDiscoveryCatalogWatcherEnabled; +import org.springframework.cloud.kubernetes.commons.discovery.ConditionalOnKubernetesCatalogEnabled; +import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; +import org.springframework.core.log.LogAccessor; + +import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.CATALOG_WATCHER_DEFAULT_DELAY; +import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.CATALOG_WATCH_PROPERTY_NAME; + +/** + * @author wind57 + */ +@Configuration(proxyBeanMethods = false) +@ConditionalOnDiscoveryEnabled +@ConditionalOnKubernetesCatalogEnabled +@ConditionalOnHttpDiscoveryCatalogWatcherEnabled +@ConditionalOnCloudPlatform(CloudPlatform.KUBERNETES) +@EnableConfigurationProperties(KubernetesDiscoveryProperties.class) +class KubernetesCatalogWatchAutoConfiguration { + + private static final LogAccessor LOG = new LogAccessor( + LogFactory.getLog(KubernetesCatalogWatchAutoConfiguration.class)); + + // this has to be a RestTemplateBuilder and not a WebClientBuilder, otherwise + // we need the webflux dependency, and it might leak into client's dependencies + // which is not always desirable. + @Bean + @ConditionalOnMissingBean + RestTemplateBuilder restTemplateBuilder() { + return new RestTemplateBuilder(); + } + + @Bean + @ConditionalOnMissingBean + KubernetesCatalogWatch kubernetesCatalogWatch(RestTemplateBuilder builder, KubernetesDiscoveryProperties properties, + Environment environment) { + + String watchDelay = environment.getProperty(CATALOG_WATCH_PROPERTY_NAME); + if (watchDelay != null) { + LOG.debug("using delay : " + watchDelay); + } + else { + LOG.debug("using default watch delay : " + CATALOG_WATCHER_DEFAULT_DELAY); + } + + return new KubernetesCatalogWatch(builder, properties); + } + +} diff --git a/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesDiscoveryClientBlockingAutoConfiguration.java b/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesDiscoveryClientBlockingAutoConfiguration.java index da4512f6..6865cd7c 100644 --- a/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesDiscoveryClientBlockingAutoConfiguration.java +++ b/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesDiscoveryClientBlockingAutoConfiguration.java @@ -28,7 +28,6 @@ import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscover import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.web.client.RestTemplate; /** * @author wind57 @@ -41,15 +40,15 @@ class KubernetesDiscoveryClientBlockingAutoConfiguration { @Bean @ConditionalOnMissingBean - RestTemplate restTemplate() { - return new RestTemplateBuilder().build(); + RestTemplateBuilder restTemplateBuilder() { + return new RestTemplateBuilder(); } @Bean @ConditionalOnMissingBean - KubernetesDiscoveryClient kubernetesDiscoveryClient(RestTemplate restTemplate, + KubernetesDiscoveryClient kubernetesDiscoveryClient(RestTemplateBuilder restTemplateBuilder, KubernetesDiscoveryProperties properties) { - return new KubernetesDiscoveryClient(restTemplate, properties); + return new KubernetesDiscoveryClient(restTemplateBuilder.build(), properties); } @Bean diff --git a/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesReactiveDiscoveryClient.java b/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesReactiveDiscoveryClient.java index c9376dc3..a5d346e6 100644 --- a/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesReactiveDiscoveryClient.java +++ b/spring-cloud-kubernetes-discovery/src/main/java/org/springframework/cloud/kubernetes/discovery/KubernetesReactiveDiscoveryClient.java @@ -40,14 +40,14 @@ public class KubernetesReactiveDiscoveryClient implements ReactiveDiscoveryClien if (!StringUtils.hasText(properties.getDiscoveryServerUrl())) { throw new DiscoveryServerUrlInvalidException(); } - this.webClient = webClientBuilder.baseUrl(properties.getDiscoveryServerUrl()).build(); + webClient = webClientBuilder.baseUrl(properties.getDiscoveryServerUrl()).build(); } KubernetesReactiveDiscoveryClient(WebClient.Builder webClientBuilder, KubernetesDiscoveryProperties properties) { if (!StringUtils.hasText(properties.discoveryServerUrl())) { throw new DiscoveryServerUrlInvalidException(); } - this.webClient = webClientBuilder.baseUrl(properties.discoveryServerUrl()).build(); + webClient = webClientBuilder.baseUrl(properties.discoveryServerUrl()).build(); } @Override diff --git a/spring-cloud-kubernetes-discovery/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/spring-cloud-kubernetes-discovery/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index 67bac3b3..5b99b7c5 100644 --- a/spring-cloud-kubernetes-discovery/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ b/spring-cloud-kubernetes-discovery/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -1,2 +1,3 @@ org.springframework.cloud.kubernetes.discovery.KubernetesDiscoveryClientBlockingAutoConfiguration org.springframework.cloud.kubernetes.discovery.KubernetesDiscoveryClientReactiveAutoConfiguration +org.springframework.cloud.kubernetes.discovery.KubernetesCatalogWatchAutoConfiguration diff --git a/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatchAutoConfigurationTests.java b/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatchAutoConfigurationTests.java new file mode 100644 index 00000000..e0860333 --- /dev/null +++ b/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatchAutoConfigurationTests.java @@ -0,0 +1,133 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.discovery; + +import org.junit.jupiter.api.Test; + +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author wind57 + */ +class KubernetesCatalogWatchAutoConfigurationTests { + + private ApplicationContextRunner applicationContextRunner; + + @Test + void discoveryEnabledDefault() { + setup("spring.main.cloud-platform=KUBERNETES", "spring.cloud.config.enabled=false", + "spring.cloud.kubernetes.discovery.discovery-server-url=example.com", + "spring.cloud.kubernetes.http.discovery.catalog.watcher.enabled=true"); + applicationContextRunner.run(context -> assertThat(context).hasSingleBean(KubernetesCatalogWatch.class)); + } + + @Test + void discoveryEnabled() { + setup("spring.main.cloud-platform=KUBERNETES", "spring.cloud.config.enabled=false", + "spring.cloud.discovery.enabled=true", + "spring.cloud.kubernetes.discovery.discovery-server-url=example.com", + "spring.cloud.kubernetes.http.discovery.catalog.watcher.enabled=true"); + applicationContextRunner.run(context -> assertThat(context).hasSingleBean(KubernetesCatalogWatch.class)); + } + + @Test + void discoveryDisabled() { + setup("spring.main.cloud-platform=KUBERNETES", "spring.cloud.config.enabled=false", + "spring.cloud.discovery.enabled=false", + "spring.cloud.kubernetes.discovery.discovery-server-url=example.com"); + applicationContextRunner.run(context -> assertThat(context).doesNotHaveBean(KubernetesCatalogWatch.class)); + } + + @Test + void kubernetesDiscoveryEnabled() { + setup("spring.main.cloud-platform=KUBERNETES", "spring.cloud.config.enabled=false", + "spring.cloud.kubernetes.discovery.enabled=true", + "spring.cloud.kubernetes.http.discovery.catalog.watcher.enabled=true", + "spring.cloud.kubernetes.discovery.discovery-server-url=example.com"); + applicationContextRunner.run(context -> assertThat(context).hasSingleBean(KubernetesCatalogWatch.class)); + } + + // disabling discovery has no impact on the catalog watch. + @Test + void kubernetesDiscoveryDisabled() { + setup("spring.main.cloud-platform=KUBERNETES", "spring.cloud.config.enabled=false", + "spring.cloud.kubernetes.discovery.enabled=false", + "spring.cloud.kubernetes.discovery.discovery-server-url=example.com", + "spring.cloud.kubernetes.http.discovery.catalog.watcher.enabled=true"); + applicationContextRunner.run(context -> assertThat(context).hasSingleBean(KubernetesCatalogWatch.class)); + } + + /** + * both blocking and reactive configs are disabled, should not influence catalog + * watcher in any way. + */ + @Test + void disableBlockingAndReactive() { + setup("spring.main.cloud-platform=KUBERNETES", "spring.cloud.config.enabled=false", + "spring.cloud.discovery.blocking.enabled=false", "spring.cloud.discovery.reactive.enabled=false", + "spring.cloud.kubernetes.http.discovery.catalog.watcher.enabled=true"); + applicationContextRunner.run(context -> { + assertThat(context).hasSingleBean(KubernetesCatalogWatch.class); + assertThat(context).doesNotHaveBean(KubernetesDiscoveryClient.class); + assertThat(context).doesNotHaveBean(KubernetesReactiveDiscoveryClient.class); + }); + } + + /** + * spring.cloud.kubernetes.discovery.enabled is false, but does not influence catalog + * watcher. + */ + @Test + void disableKubernetesDiscovery() { + setup("spring.main.cloud-platform=KUBERNETES", "spring.cloud.config.enabled=false", + "spring.cloud.kubernetes.discovery.enabled=false", + "spring.cloud.kubernetes.http.discovery.catalog.watcher.enabled=true"); + applicationContextRunner.run(context -> { + assertThat(context).hasSingleBean(KubernetesCatalogWatch.class); + assertThat(context).doesNotHaveBean(KubernetesDiscoveryClient.class); + assertThat(context).doesNotHaveBean(KubernetesReactiveDiscoveryClient.class); + }); + } + + /** + * spring.cloud.kubernetes.http.discovery.client.catalog.watcher.enabled is false, as + * such catalog watcher is not present. + */ + @Test + void disableHttpDiscoveryClientCatalogWatcher() { + setup("spring.main.cloud-platform=KUBERNETES", "spring.cloud.config.enabled=true", + "spring.cloud.kubernetes.discovery.enabled=false", + "spring.cloud.kubernetes.http.discovery.catalog.watcher.enabled=false"); + applicationContextRunner.run(context -> { + assertThat(context).doesNotHaveBean(KubernetesCatalogWatch.class); + assertThat(context).doesNotHaveBean(KubernetesDiscoveryClient.class); + assertThat(context).doesNotHaveBean(KubernetesReactiveDiscoveryClient.class); + }); + } + + private void setup(String... properties) { + applicationContextRunner = new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(KubernetesCatalogWatchAutoConfiguration.class, + KubernetesDiscoveryClientBlockingAutoConfiguration.class, + KubernetesDiscoveryClientReactiveAutoConfiguration.class)) + .withPropertyValues(properties); + } + +} diff --git a/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatchTests.java b/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatchTests.java new file mode 100644 index 00000000..689a07e1 --- /dev/null +++ b/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/KubernetesCatalogWatchTests.java @@ -0,0 +1,188 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.discovery; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.cloud.client.discovery.event.HeartbeatEvent; +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties; +import org.springframework.context.ApplicationEventPublisher; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; +import static org.mockito.Mockito.verify; + +/** + * @author wind57 + */ +class KubernetesCatalogWatchTests { + + private WireMockServer wireMockServer; + + private static final ArgumentCaptor HEARTBEAT_EVENT_ARGUMENT_CAPTOR = ArgumentCaptor + .forClass(HeartbeatEvent.class); + + private static final ApplicationEventPublisher APPLICATION_EVENT_PUBLISHER = Mockito + .mock(ApplicationEventPublisher.class); + + @AfterEach + void afterEach() { + Mockito.reset(APPLICATION_EVENT_PUBLISHER); + } + + @Test + void testSingleCycleSameAsCurrentState() { + + String body = "[]"; + + wireMockServer = new WireMockServer(options().dynamicPort()); + wireMockServer.start(); + WireMock.configureFor(wireMockServer.port()); + stubFor(get("/state") + .willReturn(aResponse().withStatus(200).withBody(body).withHeader("content-type", "application/json"))); + + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60, + false, null, Set.of(), Map.of(), null, KubernetesDiscoveryProperties.Metadata.DEFAULT, 0, false, false, + wireMockServer.baseUrl()); + + KubernetesCatalogWatch catalogWatch = new KubernetesCatalogWatch(new RestTemplateBuilder(), properties); + catalogWatch.setApplicationEventPublisher(APPLICATION_EVENT_PUBLISHER); + + catalogWatch.catalogServicesWatch(); + + Mockito.verifyNoInteractions(APPLICATION_EVENT_PUBLISHER); + + } + + @Test + @SuppressWarnings("unchecked") + void testSingleCycleDifferentCurrentState() { + + String body = """ + [ + { + "endpointName":"endpoint-name", + "namespace":"namespaceA" + } + ] + """; + + wireMockServer = new WireMockServer(options().dynamicPort()); + wireMockServer.start(); + WireMock.configureFor(wireMockServer.port()); + stubFor(get("/state") + .willReturn(aResponse().withStatus(200).withBody(body).withHeader("content-type", "application/json"))); + + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60, + false, null, Set.of(), Map.of(), null, KubernetesDiscoveryProperties.Metadata.DEFAULT, 0, false, false, + wireMockServer.baseUrl()); + + KubernetesCatalogWatch catalogWatch = new KubernetesCatalogWatch(new RestTemplateBuilder(), properties); + catalogWatch.setApplicationEventPublisher(APPLICATION_EVENT_PUBLISHER); + + catalogWatch.catalogServicesWatch(); + + verify(APPLICATION_EVENT_PUBLISHER).publishEvent(HEARTBEAT_EVENT_ARGUMENT_CAPTOR.capture()); + HeartbeatEvent event = HEARTBEAT_EVENT_ARGUMENT_CAPTOR.getValue(); + Assertions.assertEquals(event.getSource().getClass(), KubernetesCatalogWatch.class); + + List state = (List) event.getValue(); + + Assertions.assertEquals(state.size(), 1); + Assertions.assertEquals(state.get(0).namespace(), "namespaceA"); + + } + + @Test + @SuppressWarnings("unchecked") + void testTwoCyclesDifferentStates() { + + String bodyOne = """ + [ + { + "endpointName":"endpoint-name", + "namespace":"namespaceA" + } + ] + """; + + // namespace differs + String bodyTwo = """ + [ + { + "endpointName":"endpoint-name", + "namespace":"namespaceB" + } + ] + """; + + wireMockServer = new WireMockServer(options().dynamicPort()); + wireMockServer.start(); + WireMock.configureFor(wireMockServer.port()); + stubFor(get("/state").willReturn( + aResponse().withStatus(200).withBody(bodyOne).withHeader("content-type", "application/json"))); + + KubernetesDiscoveryProperties properties = new KubernetesDiscoveryProperties(true, true, Set.of(), true, 60, + false, null, Set.of(), Map.of(), null, KubernetesDiscoveryProperties.Metadata.DEFAULT, 0, false, false, + wireMockServer.baseUrl()); + + KubernetesCatalogWatch catalogWatch = new KubernetesCatalogWatch(new RestTemplateBuilder(), properties); + catalogWatch.setApplicationEventPublisher(APPLICATION_EVENT_PUBLISHER); + + catalogWatch.catalogServicesWatch(); + + verify(APPLICATION_EVENT_PUBLISHER).publishEvent(HEARTBEAT_EVENT_ARGUMENT_CAPTOR.capture()); + HeartbeatEvent eventOne = HEARTBEAT_EVENT_ARGUMENT_CAPTOR.getValue(); + Assertions.assertEquals(eventOne.getSource().getClass(), KubernetesCatalogWatch.class); + + List stateOne = (List) eventOne.getValue(); + + Assertions.assertEquals(stateOne.size(), 1); + Assertions.assertEquals(stateOne.get(0).namespace(), "namespaceA"); + + // second call + stubFor(get("/state").willReturn( + aResponse().withStatus(200).withBody(bodyTwo).withHeader("content-type", "application/json"))); + + catalogWatch.catalogServicesWatch(); + + verify(APPLICATION_EVENT_PUBLISHER, Mockito.times(2)).publishEvent(HEARTBEAT_EVENT_ARGUMENT_CAPTOR.capture()); + HeartbeatEvent eventTwo = HEARTBEAT_EVENT_ARGUMENT_CAPTOR.getValue(); + Assertions.assertEquals(eventTwo.getSource().getClass(), KubernetesCatalogWatch.class); + + List stateTwo = (List) eventTwo.getValue(); + + Assertions.assertEquals(stateTwo.size(), 1); + Assertions.assertEquals(stateTwo.get(0).namespace(), "namespaceB"); + + } + +} diff --git a/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/KubernetesDiscoveryAutoConfigurationTests.java b/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/KubernetesDiscoveryAutoConfigurationTests.java index e7e6c502..b0f9ebdd 100644 --- a/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/KubernetesDiscoveryAutoConfigurationTests.java +++ b/spring-cloud-kubernetes-discovery/src/test/java/org/springframework/cloud/kubernetes/discovery/KubernetesDiscoveryAutoConfigurationTests.java @@ -22,6 +22,7 @@ import org.springframework.boot.actuate.health.HealthIndicator; import org.springframework.boot.autoconfigure.AutoConfigurations; import org.springframework.boot.test.context.FilteredClassLoader; import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.web.client.RestTemplate; import org.springframework.web.reactive.function.client.WebClient; @@ -103,7 +104,7 @@ class KubernetesDiscoveryAutoConfigurationTests { "spring.cloud.discovery.reactive.enabled=false", "spring.cloud.kubernetes.discovery.discovery-server-url=http://k8sdiscoveryserver"); applicationContextRunner.run(context -> { - assertThat(context).hasSingleBean(RestTemplate.class); + assertThat(context).hasSingleBean(RestTemplateBuilder.class); assertThat(context).hasSingleBean(KubernetesDiscoveryClient.class); assertThat(context).getBean("indicatorInitializer").isNotNull(); @@ -147,7 +148,7 @@ class KubernetesDiscoveryAutoConfigurationTests { "spring.cloud.discovery.reactive.enabled=false", "spring.cloud.kubernetes.discovery.discovery-server-url=http://k8sdiscoveryserver"); applicationContextRunner.run(context -> { - assertThat(context).hasSingleBean(RestTemplate.class); + assertThat(context).hasSingleBean(RestTemplateBuilder.class); assertThat(context).hasSingleBean(KubernetesDiscoveryClient.class); assertThat(context).doesNotHaveBean("indicatorInitializer"); @@ -169,7 +170,7 @@ class KubernetesDiscoveryAutoConfigurationTests { "spring.cloud.kubernetes.discovery.discovery-server-url=http://k8sdiscoveryserver", "spring.cloud.discovery.client.health-indicator.enabled=false"); applicationContextRunner.run(context -> { - assertThat(context).hasSingleBean(RestTemplate.class); + assertThat(context).hasSingleBean(RestTemplateBuilder.class); assertThat(context).hasSingleBean(KubernetesDiscoveryClient.class); assertThat(context).doesNotHaveBean("indicatorInitializer"); @@ -191,7 +192,7 @@ class KubernetesDiscoveryAutoConfigurationTests { "spring.cloud.discovery.reactive.enabled=true", "spring.cloud.kubernetes.discovery.discovery-server-url=http://k8sdiscoveryserver"); applicationContextRunner.run(context -> { - assertThat(context).hasSingleBean(RestTemplate.class); + assertThat(context).hasSingleBean(RestTemplateBuilder.class); assertThat(context).hasSingleBean(KubernetesDiscoveryClient.class); assertThat(context).getBean("indicatorInitializer").isNotNull(); @@ -213,7 +214,7 @@ class KubernetesDiscoveryAutoConfigurationTests { "spring.cloud.discovery.reactive.enabled=true", "spring.cloud.kubernetes.discovery.discovery-server-url=http://k8sdiscoveryserver"); applicationContextRunner.run(context -> { - assertThat(context).hasSingleBean(RestTemplate.class); + assertThat(context).hasSingleBean(RestTemplateBuilder.class); assertThat(context).hasSingleBean(KubernetesDiscoveryClient.class); assertThat(context).getBean("indicatorInitializer").isNotNull(); diff --git a/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesCatalogWatch.java b/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesCatalogWatch.java index 7ab4e56d..805623c2 100644 --- a/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesCatalogWatch.java +++ b/spring-cloud-kubernetes-fabric8-discovery/src/main/java/org/springframework/cloud/kubernetes/fabric8/discovery/KubernetesCatalogWatch.java @@ -35,6 +35,7 @@ import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.core.log.LogAccessor; import org.springframework.scheduling.annotation.Scheduled; +import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.CATALOG_WATCH_PROPERTY_WITH_DEFAULT_VALUE; import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.DISCOVERY_GROUP; import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.DISCOVERY_VERSION; import static org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryConstants.ENDPOINT_SLICE; @@ -66,7 +67,7 @@ public class KubernetesCatalogWatch implements ApplicationEventPublisherAware { this.publisher = publisher; } - @Scheduled(fixedDelayString = "${spring.cloud.kubernetes.discovery.catalogServicesWatchDelay:30000}") + @Scheduled(fixedDelayString = "${" + CATALOG_WATCH_PROPERTY_WITH_DEFAULT_VALUE + "}") public void catalogServicesWatch() { try { diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/main/java/org/springframework/cloud/kubernetes/discoveryclient/it/HeartbeatListener.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/main/java/org/springframework/cloud/kubernetes/discoveryclient/it/HeartbeatListener.java new file mode 100644 index 00000000..099bd69b --- /dev/null +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/main/java/org/springframework/cloud/kubernetes/discoveryclient/it/HeartbeatListener.java @@ -0,0 +1,49 @@ +/* + * Copyright 2013-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.discoveryclient.it; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.commons.logging.LogFactory; + +import org.springframework.cloud.client.discovery.event.HeartbeatEvent; +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.context.ApplicationListener; +import org.springframework.core.log.LogAccessor; +import org.springframework.stereotype.Component; + +/** + * @author wind57 + */ +@Component +class HeartbeatListener implements ApplicationListener { + + private static final LogAccessor LOG = new LogAccessor(LogFactory.getLog(HeartbeatListener.class)); + + AtomicReference> state = new AtomicReference<>(List.of()); + + @Override + @SuppressWarnings("unchecked") + public void onApplicationEvent(HeartbeatEvent event) { + LOG.info("received heartbeat event"); + List state = (List) event.getValue(); + this.state.set(state); + LOG.info("state received : " + state); + } + +} diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/main/java/org/springframework/cloud/kubernetes/discoveryclient/it/KubernetesDiscoveryClientApplicationIt.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/main/java/org/springframework/cloud/kubernetes/discoveryclient/it/KubernetesDiscoveryClientApplicationIt.java index 6138c11e..6f90bae9 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/main/java/org/springframework/cloud/kubernetes/discoveryclient/it/KubernetesDiscoveryClientApplicationIt.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/main/java/org/springframework/cloud/kubernetes/discoveryclient/it/KubernetesDiscoveryClientApplicationIt.java @@ -22,6 +22,8 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.cloud.client.ServiceInstance; import org.springframework.cloud.client.discovery.DiscoveryClient; +import org.springframework.cloud.kubernetes.commons.discovery.EndpointNameAndNamespace; +import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RestController; @@ -30,13 +32,17 @@ import org.springframework.web.bind.annotation.RestController; * @author Ryan Baxter */ @SpringBootApplication +@EnableScheduling @RestController -public class KubernetesDiscoveryClientApplicationIt { +class KubernetesDiscoveryClientApplicationIt { private final DiscoveryClient discoveryClient; - public KubernetesDiscoveryClientApplicationIt(DiscoveryClient discoveryClient) { + private final HeartbeatListener heartbeatListener; + + KubernetesDiscoveryClientApplicationIt(DiscoveryClient discoveryClient, HeartbeatListener heartbeatListener) { this.discoveryClient = discoveryClient; + this.heartbeatListener = heartbeatListener; } public static void main(String[] args) { @@ -44,13 +50,18 @@ public class KubernetesDiscoveryClientApplicationIt { } @GetMapping("/services") - public List services() { + List services() { return discoveryClient.getServices(); } @GetMapping("/service/{serviceId}") - public List service(@PathVariable String serviceId) { + List service(@PathVariable String serviceId) { return discoveryClient.getInstances(serviceId); } + @GetMapping("/state") + List state() { + return heartbeatListener.state.get(); + } + } diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/java/org/springframework/cloud/kubernetes/discoveryclient/it/DiscoveryClientFilterNamespaceDelegate.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/java/org/springframework/cloud/kubernetes/discoveryclient/it/DiscoveryClientFilterNamespaceDelegate.java index 84794e28..b4b265ee 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/java/org/springframework/cloud/kubernetes/discoveryclient/it/DiscoveryClientFilterNamespaceDelegate.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/java/org/springframework/cloud/kubernetes/discoveryclient/it/DiscoveryClientFilterNamespaceDelegate.java @@ -17,14 +17,18 @@ package org.springframework.cloud.kubernetes.discoveryclient.it; import java.time.Duration; +import java.util.LinkedHashMap; import java.util.Objects; import org.assertj.core.api.Assertions; +import org.assertj.core.api.Condition; +import org.testcontainers.k3s.K3sContainer; import reactor.netty.http.client.HttpClient; import reactor.util.retry.Retry; import reactor.util.retry.RetryBackoffSpec; import org.springframework.boot.test.json.BasicJsonTester; +import org.springframework.cloud.kubernetes.integration.tests.commons.Commons; import org.springframework.http.HttpMethod; import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.web.reactive.function.client.WebClient; @@ -34,12 +38,17 @@ import org.springframework.web.reactive.function.client.WebClient; */ final class DiscoveryClientFilterNamespaceDelegate { + private DiscoveryClientFilterNamespaceDelegate() { + + } + private static final BasicJsonTester BASIC_JSON_TESTER = new BasicJsonTester( DiscoveryClientFilterNamespaceDelegate.class); - static void testNamespaceDiscoveryClient() { + static void testNamespaceDiscoveryClient(K3sContainer container) { testLoadBalancer(); testHealth(); + testForHeartbeat(container); } private static void testLoadBalancer() { @@ -76,6 +85,42 @@ final class DiscoveryClientFilterNamespaceDelegate { .extractingJsonPathStringValue("$.components.discoveryComposite.status").isEqualTo("UP"); } + private static void testForHeartbeat(K3sContainer container) { + + // 1. logs from discovery server + Commons.waitForLogStatement("using delay : 3000", container, "spring-cloud-kubernetes-discoveryserver"); + Commons.waitForLogStatement("received heartbeat event", container, "spring-cloud-kubernetes-discoveryserver"); + Commons.waitForLogStatement("state received :", container, "spring-cloud-kubernetes-discoveryserver"); + + // 2. logs from discovery client + Commons.waitForLogStatement("using delay : 3000", container, + "spring-cloud-kubernetes-k8s-client-discovery-server"); + Commons.waitForLogStatement("state received : ", container, + "spring-cloud-kubernetes-k8s-client-discovery-server"); + + // 3. heartbeat listener message + WebClient.Builder builder = builder(); + WebClient client = builder.baseUrl("http://localhost:80/discoveryclient-it/state").build(); + String result = client.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec()) + .block(); + + Condition> wireMockService = new Condition<>( + map -> map.entrySet().stream().anyMatch(en -> en.getValue().contains("service-wiremock-deployment")), + ""); + + Condition> discoveryServerService = new Condition<>( + map -> map.entrySet().stream() + .anyMatch(en -> en.getValue().contains("spring-cloud-kubernetes-k8s-client-discovery-server")), + ""); + + Assertions.assertThat(BASIC_JSON_TESTER.from(result)) + .>extractingJsonPathArrayValue("$.[*]").areAtLeastOne(wireMockService); + + Assertions.assertThat(BASIC_JSON_TESTER.from(result)) + .>extractingJsonPathArrayValue("$.[*]") + .areAtLeastOne(discoveryServerService); + } + private static WebClient.Builder builder() { return WebClient.builder().clientConnector(new ReactorClientHttpConnector(HttpClient.create())); } diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/java/org/springframework/cloud/kubernetes/discoveryclient/it/DiscoveryClientIT.java b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/java/org/springframework/cloud/kubernetes/discoveryclient/it/DiscoveryClientIT.java index 8d8ae9c2..acd215fa 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/java/org/springframework/cloud/kubernetes/discoveryclient/it/DiscoveryClientIT.java +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/java/org/springframework/cloud/kubernetes/discoveryclient/it/DiscoveryClientIT.java @@ -19,6 +19,7 @@ package org.springframework.cloud.kubernetes.discoveryclient.it; import java.time.Duration; import java.util.Map; import java.util.Objects; +import java.util.concurrent.TimeUnit; import io.kubernetes.client.openapi.apis.RbacAuthorizationV1Api; import io.kubernetes.client.openapi.models.V1ClusterRoleBinding; @@ -62,6 +63,18 @@ class DiscoveryClientIT { { "name": "SPRING_CLOUD_KUBERNETES_DISCOVERY_NAMESPACES_0", "value": "left" + }, + { + "name": "SPRING_CLOUD_KUBERNETES_DISCOVERY_CATALOGSERVICESWATCHDELAY", + "value": "3000" + }, + { + "name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_DISCOVERY", + "value": "DEBUG" + }, + { + "name": "SPRING_CLOUD_KUBERNETES_HTTP_DISCOVERY_CATALOG_WATCHER_ENABLED", + "value": "TRUE" } ] }] @@ -83,6 +96,18 @@ class DiscoveryClientIT { { "name": "SPRING_CLOUD_KUBERNETES_DISCOVERY_ALL_NAMESPACES", "value": "TRUE" + }, + { + "name": "LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_DISCOVERYSERVER", + "value": "DEBUG" + }, + { + "name": "SPRING_CLOUD_KUBERNETES_DISCOVERY_CATALOGSERVICESWATCHDELAY", + "value": "3000" + }, + { + "name": "SPRING_CLOUD_KUBERNETES_HTTP_DISCOVERY_CATALOG_WATCHER_ENABLED", + "value": "TRUE" } ] }] @@ -160,22 +185,22 @@ class DiscoveryClientIT { util.deleteNamespace(NAMESPACE_RIGHT); discoveryServer(Phase.DELETE); - discoveryIt(Phase.DELETE); + discoveryClient(Phase.DELETE); Commons.systemPrune(); } @Test void testDiscoveryClient() { - discoveryIt(Phase.CREATE); + discoveryClient(Phase.CREATE); testLoadBalancer(); testHealth(); + patchForAllNamespaces("docker.io/springcloud/spring-cloud-kubernetes-discoveryserver:" + Commons.pomVersion(), + "spring-cloud-kubernetes-discoveryserver-deployment", NAMESPACE); patchForNamespaceFilter( "docker.io/springcloud/spring-cloud-kubernetes-k8s-client-discovery-server:" + Commons.pomVersion(), "spring-cloud-kubernetes-k8s-client-discovery-server-deployment", NAMESPACE); - patchForAllNamespaces("docker.io/springcloud/spring-cloud-kubernetes-discoveryserver:" + Commons.pomVersion(), - "spring-cloud-kubernetes-discoveryserver-deployment", NAMESPACE); - testNamespaceDiscoveryClient(); + testNamespaceDiscoveryClient(K3S); } private void testLoadBalancer() { @@ -187,6 +212,31 @@ class DiscoveryClientIT { Assertions.assertThat(BASIC_JSON_TESTER.from(result)).extractingJsonPathArrayValue("$") .contains("spring-cloud-kubernetes-discoveryserver"); + + // since 'spring.cloud.kubernetes.http.discovery.client.catalog.watcher.enabled' + // is false by default, we will not receive any heartbeat events, + // simply because there are no beans registered to provide that to us. + // We assert this by doing a call to our internal /state + // endpoint, waiting 10 seconds and doing it again. Since the watch delay is set + // to 3 seconds, if there would be proper events, + // we would get a result that is different from '[]'. + + WebClient.Builder stateBuilder = builder(); + WebClient client = stateBuilder.baseUrl("http://localhost:80/discoveryclient-it/state").build(); + String stateResult = client.method(HttpMethod.GET).retrieve().bodyToMono(String.class).retryWhen(retrySpec()) + .block(); + Assertions.assertThat(BASIC_JSON_TESTER.from(stateResult)).isEqualTo("[]"); + + try { + Thread.sleep(TimeUnit.SECONDS.toMillis(10)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + + String stateResultAfter10Seconds = client.method(HttpMethod.GET).retrieve().bodyToMono(String.class) + .retryWhen(retrySpec()).block(); + Assertions.assertThat(BASIC_JSON_TESTER.from(stateResultAfter10Seconds)).isEqualTo("[]"); } void testHealth() { @@ -200,7 +250,7 @@ class DiscoveryClientIT { .extractingJsonPathStringValue("$.components.discoveryComposite.status").isEqualTo("UP"); } - private static void discoveryIt(Phase phase) { + private static void discoveryClient(Phase phase) { V1Deployment deployment = (V1Deployment) util .yaml("client/spring-cloud-kubernetes-discoveryclient-it-deployment.yaml"); V1Service service = (V1Service) util.yaml("client/spring-cloud-kubernetes-discoveryclient-it-service.yaml"); diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/resources/client/spring-cloud-kubernetes-discoveryclient-it-deployment.yaml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/resources/client/spring-cloud-kubernetes-discoveryclient-it-deployment.yaml index a6cb62c3..21ce9fd4 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/resources/client/spring-cloud-kubernetes-discoveryclient-it-deployment.yaml +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/resources/client/spring-cloud-kubernetes-discoveryclient-it-deployment.yaml @@ -30,5 +30,8 @@ spec: initialDelaySeconds: 10 periodSeconds: 2 failureThreshold: 3 + env: + - name: SPRING_CLOUD_KUBERNETES_DISCOVERY_CATALOGSERVICESWATCHDELAY + value: "3000" ports: - containerPort: 8080 diff --git a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/resources/server/spring-cloud-kubernetes-discoveryserver-deployment.yaml b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/resources/server/spring-cloud-kubernetes-discoveryserver-deployment.yaml index 3672520d..c38af74d 100644 --- a/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/resources/server/spring-cloud-kubernetes-discoveryserver-deployment.yaml +++ b/spring-cloud-kubernetes-integration-tests/spring-cloud-kubernetes-k8s-client-discovery-server/src/test/resources/server/spring-cloud-kubernetes-discoveryserver-deployment.yaml @@ -24,5 +24,10 @@ spec: httpGet: port: 8761 path: /actuator/health/liveness + env: + - name: LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_KUBERNETES_DISCOVERYSERVER + value: "DEBUG" + - name: SPRING_CLOUD_KUBERNETES_DISCOVERY_CATALOGSERVICESWATCHDELAY + value: "3000" ports: - containerPort: 8761