Instrument RestTemplate for Observability

This commit introduces Micrometer as an API dependency to the spring-web
module. Micrometer is used here to instrument `RestTemplate` and record
`Observation` for HTTP client exchanges.

This will replace Spring Boot's `MetricsClientHttpRequestInterceptor`
which uses the request interceptor contract for instrumentation.
This approach is limited as measurements and tags aren't always precise
and overhead is more important than a direct instrumentation.

See gh-28341

spring-web/src/main/java/org/springframework/http/client/observation/ClientHttpObservation.java

@@ -0,0 +1,137 @@
+/*
+ * Copyright 2002-2022 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.http.client.observation;
+
+import io.micrometer.common.docs.KeyName;
+import io.micrometer.observation.Observation;
+import io.micrometer.observation.ObservationConvention;
+import io.micrometer.observation.docs.DocumentedObservation;
+
+import org.springframework.http.client.ClientHttpRequestFactory;
+
+
+/**
+ * Documented {@link io.micrometer.common.KeyValue KeyValues} for {@link ClientHttpRequestFactory HTTP client observations}.
+ * <p>This class is used by automated tools to document KeyValues attached to the HTTP client observations.
+ * @author Brian Clozel
+ * @since 6.0
+ */
+public enum ClientHttpObservation implements DocumentedObservation {
+
+	/**
+	 * Observation created for a client HTTP exchange.
+	 */
+	HTTP_REQUEST {
+		@Override
+		public Class<? extends ObservationConvention<? extends Observation.Context>> getDefaultConvention() {
+			return DefaultClientHttpObservationConvention.class;
+		}
+
+		@Override
+		public KeyName[] getLowCardinalityKeyNames() {
+			return LowCardinalityKeyNames.values();
+		}
+
+		@Override
+		public KeyName[] getHighCardinalityKeyNames() {
+			return HighCardinalityKeyNames.values();
+		}
+
+	};
+
+	public enum LowCardinalityKeyNames implements KeyName {
+
+		/**
+		 * Name of HTTP request method or {@code "None"} if the request could not be created.
+		 */
+		METHOD {
+			@Override
+			public String asString() {
+				return "method";
+			}
+
+		},
+
+		/**
+		 * URI template used for HTTP request, or {@code ""} if none was provided.
+		 */
+		URI {
+			@Override
+			public String asString() {
+				return "uri";
+			}
+		},
+
+		/**
+		 * HTTP response raw status code, or {@code "IO_ERROR"} in case of {@code IOException},
+		 * or {@code "CLIENT_ERROR"} if no response was received.
+		 */
+		STATUS {
+			@Override
+			public String asString() {
+				return "status";
+			}
+		},
+
+		/**
+		 * Name of the exception thrown during the exchange, or {@code "None"} if no exception happened.
+		 */
+		EXCEPTION {
+			@Override
+			public String asString() {
+				return "exception";
+			}
+		},
+
+		/**
+		 * Outcome of the HTTP client exchange.
+		 * @see org.springframework.http.HttpStatus.Series
+		 */
+		OUTCOME {
+			@Override
+			public String asString() {
+				return "outcome";
+			}
+		}
+
+	}
+
+	public enum HighCardinalityKeyNames implements KeyName {
+
+		/**
+		 * HTTP request URI.
+		 */
+		URI_EXPANDED {
+			@Override
+			public String asString() {
+				return "uri.expanded";
+			}
+		},
+
+		/**
+		 * Client name derived from the request URI host.
+		 */
+		CLIENT_NAME {
+			@Override
+			public String asString() {
+				return "client.name";
+			}
+		}
+
+	}
+
+}

spring-web/src/main/java/org/springframework/http/client/observation/ClientHttpObservationContext.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2002-2022 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.http.client.observation;
+
+import io.micrometer.observation.transport.RequestReplySenderContext;
+
+import org.springframework.http.client.ClientHttpRequest;
+import org.springframework.http.client.ClientHttpRequestFactory;
+import org.springframework.http.client.ClientHttpResponse;
+import org.springframework.lang.Nullable;
+
+/**
+ * Context that holds information for metadata collection
+ * during the {@link ClientHttpRequestFactory client HTTP} observations.
+ * <p>This context also extends {@link RequestReplySenderContext} for propagating tracing
+ * information with the HTTP client exchange.
+ * @author Brian Clozel
+ * @since 6.0
+ */
+public class ClientHttpObservationContext extends RequestReplySenderContext<ClientHttpRequest, ClientHttpResponse> {
+
+	@Nullable
+	private String uriTemplate;
+
+	public ClientHttpObservationContext() {
+		super(ClientHttpObservationContext::setRequestHeader);
+	}
+
+	private static void setRequestHeader(@Nullable ClientHttpRequest request, String name, String value) {
+		if (request != null) {
+			request.getHeaders().set(name, value);
+		}
+	}
+
+	/**
+	 * Return the URI template used for the current client exchange, {@code null} if none was used.
+	 */
+	@Nullable
+	public String getUriTemplate() {
+		return this.uriTemplate;
+	}
+
+	/**
+	 * Set the URI template used for the current client exchange.
+	 */
+	public void setUriTemplate(@Nullable String uriTemplate) {
+		this.uriTemplate = uriTemplate;
+	}
+
+}

spring-web/src/main/java/org/springframework/http/client/observation/ClientHttpObservationConvention.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2002-2022 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.http.client.observation;
+
+import io.micrometer.observation.Observation;
+import io.micrometer.observation.ObservationConvention;
+
+/**
+ * Interface for an {@link ObservationConvention} related to RestTemplate HTTP exchanges.
+ * @author Brian Clozel
+ * @since 6.0
+ */
+public interface ClientHttpObservationConvention extends ObservationConvention<ClientHttpObservationContext> {
+
+	@Override
+	default boolean supportsContext(Observation.Context context) {
+		return context instanceof ClientHttpObservationContext;
+	}
+
+}

spring-web/src/main/java/org/springframework/http/client/observation/DefaultClientHttpObservationConvention.java

@@ -0,0 +1,152 @@
+/*
+ * Copyright 2002-2022 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.http.client.observation;
+
+import java.io.IOException;
+
+import io.micrometer.common.KeyValue;
+import io.micrometer.common.KeyValues;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.http.client.ClientHttpResponse;
+import org.springframework.lang.Nullable;
+import org.springframework.util.StringUtils;
+
+/**
+ * Default implementation for a {@link ClientHttpObservationConvention},
+ * extracting information from the {@link ClientHttpObservationContext}.
+ *
+ * @author Brian Clozel
+ * @since 6.0
+ */
+public class DefaultClientHttpObservationConvention implements ClientHttpObservationConvention {
+
+	private static final String DEFAULT_NAME = "http.client.requests";
+
+	private static final KeyValue URI_NONE = KeyValue.of(ClientHttpObservation.LowCardinalityKeyNames.URI, "none");
+
+	private static final KeyValue METHOD_NONE = KeyValue.of(ClientHttpObservation.LowCardinalityKeyNames.METHOD, "none");
+
+	private static final KeyValue EXCEPTION_NONE = KeyValue.of(ClientHttpObservation.LowCardinalityKeyNames.EXCEPTION, "none");
+
+	private static final KeyValue OUTCOME_UNKNOWN = KeyValue.of(ClientHttpObservation.LowCardinalityKeyNames.OUTCOME, "UNKNOWN");
+
+	private static final KeyValue URI_EXPANDED_NONE = KeyValue.of(ClientHttpObservation.HighCardinalityKeyNames.URI_EXPANDED, "none");
+
+	private final String name;
+
+	/**
+	 * Create a convention with the default name {@code "http.client.requests"}.
+	 */
+	public DefaultClientHttpObservationConvention() {
+		this(DEFAULT_NAME);
+	}
+
+	/**
+	 * Create a convention with a custom name.
+	 * @param name the observation name
+	 */
+	public DefaultClientHttpObservationConvention(String name) {
+		this.name = name;
+	}
+
+	@Override
+	public String getName() {
+		return this.name;
+	}
+
+	@Override
+	public KeyValues getLowCardinalityKeyValues(ClientHttpObservationContext context) {
+		return KeyValues.of(uri(context), method(context), status(context), exception(context), outcome(context));
+	}
+
+	protected KeyValue uri(ClientHttpObservationContext context) {
+		if (context.getUriTemplate() != null) {
+			return KeyValue.of(ClientHttpObservation.LowCardinalityKeyNames.URI, context.getUriTemplate());
+		}
+		return URI_NONE;
+	}
+
+	protected KeyValue method(ClientHttpObservationContext context) {
+		if (context.getCarrier() != null) {
+			return KeyValue.of(ClientHttpObservation.LowCardinalityKeyNames.METHOD, context.getCarrier().getMethod().name());
+		}
+		else {
+			return METHOD_NONE;
+		}
+	}
+
+	protected KeyValue status(ClientHttpObservationContext context) {
+		return KeyValue.of(ClientHttpObservation.LowCardinalityKeyNames.STATUS, getStatusMessage(context.getResponse()));
+	}
+
+	private String getStatusMessage(@Nullable ClientHttpResponse response) {
+		try {
+			if (response == null) {
+				return "CLIENT_ERROR";
+			}
+			return String.valueOf(response.getStatusCode().value());
+		}
+		catch (IOException ex) {
+			return "IO_ERROR";
+		}
+	}
+
+	protected KeyValue exception(ClientHttpObservationContext context) {
+		return context.getError().map(exception -> {
+			String simpleName = exception.getClass().getSimpleName();
+			return KeyValue.of(ClientHttpObservation.LowCardinalityKeyNames.EXCEPTION,
+					StringUtils.hasText(simpleName) ? simpleName : exception.getClass().getName());
+		}).orElse(EXCEPTION_NONE);
+	}
+
+	protected static KeyValue outcome(ClientHttpObservationContext context) {
+		try {
+			if (context.getResponse() != null) {
+				HttpStatus status = HttpStatus.resolve(context.getResponse().getStatusCode().value());
+				if (status != null) {
+					return KeyValue.of(ClientHttpObservation.LowCardinalityKeyNames.OUTCOME, status.series().name());
+				}
+			}
+		}
+		catch (IOException ex) {
+			// Continue
+		}
+		return OUTCOME_UNKNOWN;
+	}
+
+	@Override
+	public KeyValues getHighCardinalityKeyValues(ClientHttpObservationContext context) {
+		return KeyValues.of(requestUri(context), clientName(context));
+	}
+
+	protected KeyValue requestUri(ClientHttpObservationContext context) {
+		if (context.getCarrier() != null) {
+			return KeyValue.of(ClientHttpObservation.HighCardinalityKeyNames.URI_EXPANDED, context.getCarrier().getURI().toASCIIString());
+		}
+		return URI_EXPANDED_NONE;
+	}
+
+	protected KeyValue clientName(ClientHttpObservationContext context) {
+		String host = "none";
+		if (context.getCarrier() != null && context.getCarrier().getURI().getHost() != null) {
+			host = context.getCarrier().getURI().getHost();
+		}
+		return KeyValue.of(ClientHttpObservation.HighCardinalityKeyNames.CLIENT_NAME, host);
+	}
+
+}

spring-web/src/main/java/org/springframework/http/client/observation/package-info.java

@@ -0,0 +1,10 @@
+/**
+ * This package provides support for client HTTP
+ * {@link io.micrometer.observation.Observation}.
+ */
+@NonNullApi
+@NonNullFields
+package org.springframework.http.client.observation;
+
+import org.springframework.lang.NonNullApi;
+import org.springframework.lang.NonNullFields;

spring-web/src/main/java/org/springframework/web/client/RestTemplate.java

@@ -28,6 +28,9 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import io.micrometer.observation.Observation;
+import io.micrometer.observation.ObservationRegistry;
+
 import org.springframework.core.ParameterizedTypeReference;
 import org.springframework.core.SpringProperties;
 import org.springframework.http.HttpEntity;
@@ -40,6 +43,10 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.http.client.ClientHttpRequest;
 import org.springframework.http.client.ClientHttpRequestFactory;
 import org.springframework.http.client.ClientHttpResponse;
+import org.springframework.http.client.observation.ClientHttpObservation;
+import org.springframework.http.client.observation.ClientHttpObservationContext;
+import org.springframework.http.client.observation.ClientHttpObservationConvention;
+import org.springframework.http.client.observation.DefaultClientHttpObservationConvention;
 import org.springframework.http.client.support.InterceptingHttpAccessor;
 import org.springframework.http.converter.ByteArrayHttpMessageConverter;
 import org.springframework.http.converter.GenericHttpMessageConverter;
@@ -119,6 +126,8 @@ public class RestTemplate extends InterceptingHttpAccessor implements RestOperat
 
 	private static final boolean kotlinSerializationJsonPresent;
 
+	private static final ClientHttpObservationConvention DEFAULT_OBSERVATION_CONVENTION = new DefaultClientHttpObservationConvention();
+
 	static {
 		ClassLoader classLoader = RestTemplate.class.getClassLoader();
 		romePresent = ClassUtils.isPresent("com.rometools.rome.feed.WireFeed", classLoader);
@@ -142,6 +151,11 @@ public class RestTemplate extends InterceptingHttpAccessor implements RestOperat
 
 	private final ResponseExtractor<HttpHeaders> headersExtractor = new HeadersExtractor();
 
+	private ObservationRegistry observationRegistry = ObservationRegistry.NOOP;
+
+	@Nullable
+	private ClientHttpObservationConvention observationConvention;
+
 
 	/**
 	 * Create a new instance of the {@link RestTemplate} using default settings.
@@ -323,6 +337,30 @@ public class RestTemplate extends InterceptingHttpAccessor implements RestOperat
 		return this.uriTemplateHandler;
 	}
 
+	/**
+	 * Configure an {@link ObservationRegistry} for collecting spans and metrics
+	 * for request execution. By default, {@link Observation} are No-Ops.
+	 * @param observationRegistry the observation registry to use
+	 * @since 6.0
+	 */
+	public void setObservationRegistry(ObservationRegistry observationRegistry) {
+		Assert.notNull(observationRegistry, "observationRegistry must not be null");
+		this.observationRegistry = observationRegistry;
+	}
+
+	/**
+	 * Configure an {@link Observation.ObservationConvention} that sets the name of the
+	 * {@link Observation observation} as well as its {@link io.micrometer.common.KeyValues}
+	 * extracted from the {@link ClientHttpObservationContext}.
+	 * If none set, the {@link DefaultClientHttpObservationConvention default convention} will be used.
+	 * @param observationConvention the observation convention to use
+	 * @since 6.0
+	 * @see #setObservationRegistry(ObservationRegistry)
+	 */
+	public void setObservationConvention(ClientHttpObservationConvention observationConvention) {
+		Assert.notNull(observationConvention, "observationConvention must not be null");
+		this.observationConvention = observationConvention;
+	}
 
 	// GET
 
@@ -658,7 +696,7 @@ public class RestTemplate extends InterceptingHttpAccessor implements RestOperat
 
 		RequestCallback requestCallback = httpEntityCallback(entity, responseType);
 		ResponseExtractor<ResponseEntity<T>> responseExtractor = responseEntityExtractor(responseType);
-		return nonNull(doExecute(resolveUrl(entity), entity.getMethod(), requestCallback, responseExtractor));
+		return nonNull(doExecute(resolveUrl(entity), resolveUriTemplate(entity), entity.getMethod(), requestCallback, responseExtractor));
 	}
 
 	@Override
@@ -668,7 +706,7 @@ public class RestTemplate extends InterceptingHttpAccessor implements RestOperat
 		Type type = responseType.getType();
 		RequestCallback requestCallback = httpEntityCallback(entity, type);
 		ResponseExtractor<ResponseEntity<T>> responseExtractor = responseEntityExtractor(type);
-		return nonNull(doExecute(resolveUrl(entity), entity.getMethod(), requestCallback, responseExtractor));
+		return nonNull(doExecute(resolveUrl(entity), resolveUriTemplate(entity), entity.getMethod(), requestCallback, responseExtractor));
 	}
 
 	private URI resolveUrl(RequestEntity<?> entity) {
@@ -689,6 +727,16 @@ public class RestTemplate extends InterceptingHttpAccessor implements RestOperat
 		}
 	}
 
+	@Nullable
+	private String resolveUriTemplate(RequestEntity<?> entity) {
+		if (entity instanceof RequestEntity.UriTemplateRequestEntity<?> templated) {
+			return templated.getUriTemplate();
+		}
+		else {
+			return null;
+		}
+	}
+
 
 	// General execution
 
@@ -705,11 +753,11 @@ public class RestTemplate extends InterceptingHttpAccessor implements RestOperat
 	 */
 	@Override
 	@Nullable
-	public <T> T execute(String url, HttpMethod method, @Nullable RequestCallback requestCallback,
+	public <T> T execute(String uriTemplate, HttpMethod method, @Nullable RequestCallback requestCallback,
 			@Nullable ResponseExtractor<T> responseExtractor, Object... uriVariables) throws RestClientException {
 
-		URI expanded = getUriTemplateHandler().expand(url, uriVariables);
-		return doExecute(expanded, method, requestCallback, responseExtractor);
+		URI url = getUriTemplateHandler().expand(uriTemplate, uriVariables);
+		return doExecute(url, uriTemplate, method, requestCallback, responseExtractor);
 	}
 
 	/**
@@ -725,12 +773,12 @@ public class RestTemplate extends InterceptingHttpAccessor implements RestOperat
 	 */
 	@Override
 	@Nullable
-	public <T> T execute(String url, HttpMethod method, @Nullable RequestCallback requestCallback,
+	public <T> T execute(String uriTemplate, HttpMethod method, @Nullable RequestCallback requestCallback,
 			@Nullable ResponseExtractor<T> responseExtractor, Map<String, ?> uriVariables)
 			throws RestClientException {
 
-		URI expanded = getUriTemplateHandler().expand(url, uriVariables);
-		return doExecute(expanded, method, requestCallback, responseExtractor);
+		URI url = getUriTemplateHandler().expand(uriTemplate, uriVariables);
+		return doExecute(url, uriTemplate, method, requestCallback, responseExtractor);
 	}
 
 	/**
@@ -749,7 +797,7 @@ public class RestTemplate extends InterceptingHttpAccessor implements RestOperat
 	public <T> T execute(URI url, HttpMethod method, @Nullable RequestCallback requestCallback,
 			@Nullable ResponseExtractor<T> responseExtractor) throws RestClientException {
 
-		return doExecute(url, method, requestCallback, responseExtractor);
+		return doExecute(url, null, method, requestCallback, responseExtractor);
 	}
 
 	/**
@@ -761,20 +809,49 @@ public class RestTemplate extends InterceptingHttpAccessor implements RestOperat
 	 * @param requestCallback object that prepares the request (can be {@code null})
 	 * @param responseExtractor object that extracts the return value from the response (can be {@code null})
 	 * @return an arbitrary object, as returned by the {@link ResponseExtractor}
+	 * @deprecated in favor of {@link #doExecute(URI, String, HttpMethod, RequestCallback, ResponseExtractor)}
 	 */
 	@Nullable
+	@Deprecated
 	protected <T> T doExecute(URI url, @Nullable HttpMethod method, @Nullable RequestCallback requestCallback,
 			@Nullable ResponseExtractor<T> responseExtractor) throws RestClientException {
 
-		Assert.notNull(url, "URI is required");
+		return doExecute(url, null, method, requestCallback, responseExtractor);
+	}
+
+	/**
+	 * Execute the given method on the provided URI.
+	 * <p>The {@link ClientHttpRequest} is processed using the {@link RequestCallback};
+	 * the response with the {@link ResponseExtractor}.
+	 * @param url the fully-expanded URL to connect to
+	 * @param uriTemplate the URI template that was used for creating the expanded URL
+	 * @param method the HTTP method to execute (GET, POST, etc.)
+	 * @param requestCallback object that prepares the request (can be {@code null})
+	 * @param responseExtractor object that extracts the return value from the response (can be {@code null})
+	 * @return an arbitrary object, as returned by the {@link ResponseExtractor}
+	 */
+	@Nullable
+	@SuppressWarnings("try")
+	protected <T> T doExecute(URI url, @Nullable String uriTemplate, @Nullable HttpMethod method, @Nullable RequestCallback requestCallback,
+			@Nullable ResponseExtractor<T> responseExtractor) throws RestClientException {
+
+		Assert.notNull(url, "url is required");
 		Assert.notNull(method, "HttpMethod is required");
+		ClientHttpObservationContext observationContext = new ClientHttpObservationContext();
+		Observation observation = ClientHttpObservation.HTTP_REQUEST.observation(this.observationConvention,
+				DEFAULT_OBSERVATION_CONVENTION, observationContext, this.observationRegistry).start();
+		observationContext.setUriTemplate(uriTemplate);
 		ClientHttpResponse response = null;
 		try {
 			ClientHttpRequest request = createRequest(url, method);
+			observationContext.setCarrier(request);
 			if (requestCallback != null) {
 				requestCallback.doWithRequest(request);
 			}
-			response = request.execute();
+			try (Observation.Scope scope = observation.openScope()) {
+				response = request.execute();
+			}
+			observationContext.setResponse(response);
 			handleResponse(url, method, response);
 			return (responseExtractor != null ? responseExtractor.extractData(response) : null);
 		}
@@ -782,13 +859,20 @@ public class RestTemplate extends InterceptingHttpAccessor implements RestOperat
 			String resource = url.toString();
 			String query = url.getRawQuery();
 			resource = (query != null ? resource.substring(0, resource.indexOf('?')) : resource);
-			throw new ResourceAccessException("I/O error on " + method.name() +
+			ResourceAccessException exception = new ResourceAccessException("I/O error on " + method.name() +
 					" request for \"" + resource + "\": " + ex.getMessage(), ex);
+			observation.error(exception);
+			throw exception;
+		}
+		catch (RestClientException exc) {
+			observation.error(exc);
+			throw exc;
 		}
 		finally {
 			if (response != null) {
 				response.close();
 			}
+			observation.stop();
 		}
 	}