Add http-request-function and processor app

* Update README
* Clean up per review
* Added function README
* Remove `HttpRequestProcessorProperties$Retry` from the whitelist
This commit is contained in:
David Turanski
2020-05-28 11:01:36 -04:00
committed by GitHub
parent c0920d9ba2
commit 1ef16b9b3e
16 changed files with 1242 additions and 2 deletions

View File

@@ -0,0 +1,59 @@
//tag::ref-doc[]
= Http Request Processor
A processor app that makes requests to an HTTP resource and emits the response body as a message payload.
== Input
=== Headers
Any Required HTTP headers must be explicitly set via the `headers` or `headers-expression` property. See examples below.
Header values may also be used to construct:
* the request body when referenced in the `body-expression` property.
* the HTTP method when referenced in the `http-method-expression` property.
* the URL when referenced in the `url-expression` property.
=== Payload
The payload is used as the request body for a POST request by default, and can be any Java type.
It should be an empty String for a GET request.
The payload may also be used to construct:
* the request body when referenced in the `body-expression` property.
* the HTTP method when referenced in the `http-method-expression` property.
* the URL when referenced in the `url-expression` property.
The underlying https://docs.spring.io/spring/docs/current/javadoc-api/org/springframework/web/reactive/function/client/WebClient.html[WebClient] supports Jackson JSON serialization to support any request and response types if necessary.
The `expected-response-type` property, `String.class` by default, may be set to any class in your application class path.
Note that user defined payload types will require adding required dependencies to your pom file.
== Output
=== Headers
No HTTP message headers are mapped to the outbound Message.
=== Payload
The raw output object is https://docs.spring.io/spring/docs/current/javadoc-api/org/springframework/http/ResponseEntity.html[ResponseEntity<?>] any of its fields (e.g., `body`, `headers`) or accessor methods (`statusCode`) may be referenced as part of the `reply-expression`.
By default the outbound Message payload is the response body.
Note that ResponseEntity (referenced by the expression `#root`) cannot be deserialized by Jackson by default, but may be rendered as a `HashMap`.
== Options
The **$$http-request$$** $$processor$$ has the following options:
== Options
//tag::configuration-properties[]
$$http.request.processor.body$$:: $$The (static) request body; if neither this nor bodyExpression is provided, the payload will be used.$$ *($$Object$$, default: `$$<none>$$`)*
$$http.request.processor.body-expression$$:: $$A SpEL expression to derive the request body from the incoming message.$$ *($$Expression$$, default: `$$<none>$$`)*
$$http.request.processor.expected-response-type$$:: $$The type used to interpret the response.$$ *($$Class<?>$$, default: `$$<none>$$`)*
$$http.request.processor.headers$$:: $$A Map of HTTP request headers.$$ *($$HttpHeaders$$, default: `$$<none>$$`)*
$$http.request.processor.headers-expression$$:: $$A SpEL expression used to derive the http headers map to use.$$ *($$Expression$$, default: `$$<none>$$`)*
$$http.request.processor.http-method$$:: $$The kind of http method to use.$$ *($$HttpMethod$$, default: `$$<none>$$`, possible values: `GET`,`HEAD`,`POST`,`PUT`,`PATCH`,`DELETE`,`OPTIONS`,`TRACE`)*
$$http.request.processor.http-method-expression$$:: $$A SpEL expression to derive the request method from the incoming message.$$ *($$Expression$$, default: `$$<none>$$`)*
$$http.request.processor.reply-expression$$:: $$A SpEL expression used to compute the final result, applied against the whole http {@link org.springframework.http.ResponseEntity}.$$ *($$Expression$$, default: `$$body$$`)*
$$http.request.processor.timeout$$:: $$Request timeout in milliseconds.$$ *($$Long$$, default: `$$30000$$`)*
$$http.request.processor.url$$:: $$The URL to issue an http request to, as a static value.$$ *($$String$$, default: `$$<none>$$`)*
$$http.request.processor.url-expression$$:: $$A SpEL expression against incoming message to determine the URL to use.$$ *($$Expression$$, default: `$$<none>$$`)*
//end::configuration-properties[]
//end::ref-doc[]

View File

@@ -0,0 +1,104 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>http-request-processor</artifactId>
<version>3.0.0-SNAPSHOT</version>
<name>http-request-processor</name>
<description>HTTP request processor apps</description>
<packaging>jar</packaging>
<parent>
<groupId>org.springframework.cloud.stream.app</groupId>
<artifactId>stream-applications-core</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath/>
</parent>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.cloud.fn</groupId>
<artifactId>http-request-function</artifactId>
<version>${java-functions.version}</version>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>mockwebserver</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-test</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-app-starter-doc-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.springframework.cloud.stream.app.plugin</groupId>
<artifactId>spring-cloud-stream-app-maven-plugin</artifactId>
<configuration>
<generatedApp>
<name>http-request</name>
<type>processor</type>
<version>${project.version}</version>
<configClass>org.springframework.cloud.stream.app.processor.http.request.HttpRequestProcessorConfiguration.class
</configClass>
<functionDefinition>httpRequestProcessor</functionDefinition>
</generatedApp>
<dependencies>
<dependency>
<groupId>org.springframework.cloud.stream.app</groupId>
<artifactId>http-request-processor</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<containerImage>
<enableMetadata>true</enableMetadata>
</containerImage>
</configuration>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<snapshots>
<enabled>true</enabled>
</snapshots>
<id>spring-snapshots</id>
<name>Spring Snapshots</name>
<url>https://repo.spring.io/libs-snapshot-local</url>
</repository>
<repository>
<snapshots>
<enabled>false</enabled>
</snapshots>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/libs-milestone-local</url>
</repository>
</repositories>
</project>

View File

@@ -0,0 +1,86 @@
/*
* Copyright 2020-2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.app.processor.http.request;
import java.util.function.Function;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.messaging.Message;
import org.springframework.web.reactive.function.client.WebClient;
import static org.springframework.cloud.fn.http.request.HttpRequestFunctionConfiguration.HttpRequestFunction;
@Configuration
@EnableConfigurationProperties({ HttpRequestProcessorProperties.class })
public class HttpRequestProcessorConfiguration {
private static Log log = LogFactory.getLog(HttpRequestProcessorConfiguration.class);
@Bean
@ConditionalOnMissingBean(WebClient.class)
public WebClient webClient() {
return WebClient.builder()
.build();
}
@Bean
HttpRequestFunctionFactory httpRequestFunctionFactory(WebClient webClient,
HttpRequestProcessorProperties properties) {
return new HttpRequestFunctionFactory(webClient, properties);
}
@Bean
Function<Message<?>, ?> httpRequestProcessor(HttpRequestFunctionFactory httpRequestFunctionFactory,
HttpRequestProcessorProperties properties) {
return message -> Mono.from(httpRequestFunctionFactory.getHttpRequestFunction(message).apply(Flux.just(message))
.map(responseEntity -> properties.getReplyExpression().getValue(responseEntity)))
.doOnError(e -> log.error(e.getMessage(), e))
.block();
}
static class HttpRequestFunctionFactory {
private final WebClient webClient;
private final HttpRequestProcessorProperties properties;
private final HttpRequestFunction instance;
HttpRequestFunctionFactory(WebClient webClient, HttpRequestProcessorProperties properties) {
this.properties = properties;
this.webClient = webClient;
this.instance = properties.usesRequestExpressions() ? null
: new HttpRequestFunction(webClient, properties);
}
HttpRequestFunction getHttpRequestFunction(Message<?> message) {
if (instance != null) {
return instance;
}
return new HttpRequestFunction(webClient, properties.evaluateFunctionProperties(message));
}
}
}

View File

@@ -0,0 +1,254 @@
/*
* Copyright 2020-2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.app.processor.http.request;
import java.util.Map;
import javax.validation.constraints.AssertTrue;
import javax.validation.constraints.NotNull;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.cloud.fn.http.request.HttpRequestFunctionProperties;
import org.springframework.cloud.fn.http.request.HttpRequestProperties;
import org.springframework.expression.Expression;
import org.springframework.expression.spel.standard.SpelExpressionParser;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.messaging.Message;
import org.springframework.validation.annotation.Validated;
/**
* Configuration properties for the Http Request Processor module.
*
* @author Waldemar Hummer
* @author Mark Fisher
* @author Christian Tzolov
* @author Artem Bilan
* @author David Turanski
*/
@Validated
@ConfigurationProperties("http.request.processor")
public class HttpRequestProcessorProperties implements HttpRequestProperties {
private static final HttpMethod DEFAULT_HTTP_METHOD = HttpMethod.GET;
private static final Class<?> DEFAULT_RESPONSE_TYPE = String.class;
/**
* The URL to issue an http request to, as a static value.
*/
private String url;
/**
* The (static) request body; if neither this nor bodyExpression is provided, the payload
* will be used.
*/
private Object body;
/**
* The kind of http method to use.
*/
private HttpMethod httpMethod = DEFAULT_HTTP_METHOD;
/**
* The type used to interpret the response.
*/
private Class<?> expectedResponseType = DEFAULT_RESPONSE_TYPE;
/**
* Request timeout in milliseconds.
*/
private long timeout = 30_000;
/**
* A Map of HTTP request headers.
*/
private HttpHeaders headers = new HttpHeaders();
/**
* A SpEL expression against incoming message to determine the URL to use.
*/
private Expression urlExpression;
/**
* A SpEL expression to derive the request method from the incoming message.
*/
private Expression httpMethodExpression;
/**
* A SpEL expression to derive the request body from the incoming message.
*/
private Expression bodyExpression;
/**
* A SpEL expression used to derive the http headers map to use.
*/
private Expression headersExpression;
/**
* A SpEL expression used to compute the final result, applied against the whole http
* {@link org.springframework.http.ResponseEntity}.
*/
private Expression replyExpression = new SpelExpressionParser().parseExpression("body");
public Expression getUrlExpression() {
return urlExpression;
}
public void setUrlExpression(Expression urlExpression) {
this.urlExpression = urlExpression;
}
public Expression getHttpMethodExpression() {
return httpMethodExpression;
}
public void setHttpMethodExpression(Expression httpMethodExpression) {
this.setHttpMethod(null);
this.httpMethodExpression = httpMethodExpression;
}
@Override
public String getUrl() {
return this.url;
}
public void setUrl(String url) {
this.url = url;
}
@Override
public HttpMethod getHttpMethod() {
return httpMethod;
}
public void setHttpMethod(HttpMethod httpMethod) {
this.httpMethod = httpMethod;
}
@NotNull
@Override
public Class<?> getExpectedResponseType() {
return expectedResponseType;
}
public void setExpectedResponseType(Class<?> expectedResponseType) {
this.expectedResponseType = expectedResponseType;
}
@Override
public Object getBody() {
return body;
}
public void setBody(Object body) {
this.body = body;
}
@Override
public long getTimeout() {
return this.timeout;
}
public void setTimeout(long timeout) {
this.timeout = timeout;
}
@Override
public HttpHeaders getHeaders() {
return headers;
}
public void setHeaders(HttpHeaders headers) {
this.headers = headers;
}
public Expression getBodyExpression() {
return bodyExpression;
}
public void setBodyExpression(Expression bodyExpression) {
this.bodyExpression = bodyExpression;
}
public Expression getHeadersExpression() {
return headersExpression;
}
public void setHeadersExpression(Expression headersExpression) {
this.headersExpression = headersExpression;
}
@NotNull
public Expression getReplyExpression() {
return replyExpression;
}
public void setReplyExpression(Expression replyExpression) {
this.replyExpression = replyExpression;
}
@AssertTrue(message = "Exactly one of 'url' or 'urlExpression' is required")
public boolean isExactlyOneUrl() {
return getUrl() == null ^ urlExpression == null;
}
@AssertTrue(message = "At most one of 'body' or 'bodyExpression' is allowed")
public boolean isAtMostOneBody() {
return getBody() == null || bodyExpression == null;
}
@AssertTrue(message = "At most one of 'httpMethod' or 'httpMethodExpression' is allowed")
public boolean isAtMostOneHttpMethod() {
return getHttpMethod() == null || httpMethodExpression == null;
}
public boolean usesRequestExpressions() {
return headersExpression != null ||
bodyExpression != null ||
httpMethodExpression != null ||
urlExpression != null;
}
HttpRequestFunctionProperties evaluateFunctionProperties(Message<?> message) {
HttpRequestFunctionProperties properties = new HttpRequestFunctionProperties();
properties.setUrl(urlExpression != null ? urlExpression.getValue(message, String.class) : getUrl());
properties.setBody(bodyExpression != null ? bodyExpression.getValue(message) : getBody());
properties.setHttpMethod(httpMethodExpression != null ? httpMethodExpression.getValue(message, HttpMethod.class)
: getHttpMethod());
HttpHeaders headers = new HttpHeaders();
headers.addAll(getHeaders());
if (headersExpression != null) {
Map<?, ?> headersMap = headersExpression.getValue(message, Map.class);
for (Map.Entry<?, ?> header : headersMap.entrySet()) {
if (header.getKey() != null && header.getValue() != null) {
headers.add(header.getKey().toString(),
header.getValue().toString());
}
}
}
properties.setHeaders(headers);
properties.setTimeout(getTimeout());
properties.setExpectedResponseType(getExpectedResponseType());
return properties;
}
}

View File

@@ -0,0 +1 @@
configuration-properties.classes=org.springframework.cloud.stream.app.processor.http.request.HttpRequestProcessorProperties

View File

@@ -0,0 +1,196 @@
/*
* Copyright 2020-2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.app.processor.http.request;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import com.fasterxml.jackson.databind.ObjectMapper;
import okhttp3.mockwebserver.Dispatcher;
import okhttp3.mockwebserver.MockResponse;
import okhttp3.mockwebserver.MockWebServer;
import okhttp3.mockwebserver.RecordedRequest;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.springframework.cloud.stream.binder.test.InputDestination;
import org.springframework.cloud.stream.binder.test.OutputDestination;
import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageHeaders;
import org.springframework.messaging.support.MessageBuilder;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatIllegalStateException;
public class HttpRequestProcessorTests {
private static MockWebServer server;
private ApplicationContextRunner applicationContextRunner;
@BeforeEach
void setup() {
applicationContextRunner = new ApplicationContextRunner().withUserConfiguration(
TestChannelBinderConfiguration.getCompleteConfiguration(HttpRequestProcessorApp.class));
}
@BeforeAll
static void startServer() {
server = new MockWebServer();
server.setDispatcher(new Dispatcher() {
@Override
public MockResponse dispatch(RecordedRequest recordedRequest) {
return new MockResponse()
.setBody(recordedRequest.getBody())
.setResponseCode(HttpStatus.OK.value());
}
});
}
@AfterAll
static void shutdownServer() throws IOException {
server.shutdown();
}
private String url() {
return String.format("http://localhost:%d", server.getPort());
}
@Test
void requestUsingExpressions() throws IOException {
applicationContextRunner
.withPropertyValues(
"http.request.processor.url-expression=headers['url']",
"http.request.processor.http-method-expression=headers['method']",
"http.request.processor.body-expression=headers['body']",
"http.request.processor.headers-expression={Accept:'application/json'}",
"http.request.processor.reply-expression=#root")
.run(context -> {
Message<?> message = MessageBuilder.withPayload("")
.setHeader("url", url())
.setHeader("method", "POST")
.setHeader("body", "{\"hello\":\"world\"}")
.build();
InputDestination inputDestination = context.getBean(InputDestination.class);
OutputDestination outputDestination = context.getBean(OutputDestination.class);
ObjectMapper objectMapper = context.getBean(ObjectMapper.class);
inputDestination.send(message);
Message<byte[]> reply = outputDestination.receive(100);
// Cannot deserialize ResponseEntity directly.
Map responseEntityAsMap = objectMapper.readValue(reply.getPayload(), HashMap.class);
System.out.println(responseEntityAsMap);
assertThat(responseEntityAsMap.get("statusCode")).isEqualTo("OK");
assertThat(responseEntityAsMap.get("body")).isEqualTo(message.getHeaders().get("body"));
assertThat(reply.getHeaders().get(MessageHeaders.CONTENT_TYPE))
.isEqualTo(MediaType.APPLICATION_JSON);
});
}
@Test
void requestUsingReturnType() throws IOException {
applicationContextRunner
.withPropertyValues(
"http.request.processor.url=" + url(),
"http.request.processor.httpMethod=POST",
"http.request.processor.headers[Accept]=application/octet-stream",
"http.request.processor.expectedResponseType=byte[]",
"spring.cloud.stream.bindings.httpRequestProcessor-out-0.contentType=application/octet-stream")
.run(context -> {
Message<?> message = MessageBuilder.withPayload("hello")
.build();
InputDestination inputDestination = context.getBean(InputDestination.class);
OutputDestination outputDestination = context.getBean(OutputDestination.class);
inputDestination.send(message);
Message<byte[]> reply = outputDestination.receive(100);
assertThat(new String(reply.getPayload())).isEqualTo(message.getPayload());
assertThat(reply.getHeaders().get(MessageHeaders.CONTENT_TYPE))
.isEqualTo(MediaType.APPLICATION_OCTET_STREAM);
});
}
@Test
void requestUsingJsonPathMethodExpression() throws IOException {
applicationContextRunner
.withPropertyValues(
"http.request.processor.url=" + url(),
"http.request.processor.httpMethodExpression=#jsonPath(payload,'$.myMethod')")
.run(context -> {
Message<?> message = MessageBuilder
.withPayload("{\"name\":\"Fred\",\"age\":41, \"myMethod\":\"POST\"}")
.build();
InputDestination inputDestination = context.getBean(InputDestination.class);
OutputDestination outputDestination = context.getBean(OutputDestination.class);
inputDestination.send(message);
Message<byte[]> reply = outputDestination.receive(100);
assertThat(new String(reply.getPayload())).isEqualTo(message.getPayload());
});
}
@Test
void cannotSpecifyBothUrlandUrlExpression() {
applicationContextRunner
.withPropertyValues("http.request.processor.url=http://example.com",
"http.request.processor.url-expression=headers['url']")
.run(context -> {
assertThatIllegalStateException().isThrownBy(() -> {
context.start();
});
});
}
@Test
void cannotSpecifyBothHttpMethosdandHttpMethodExpression() {
applicationContextRunner
.withPropertyValues("http.request.processor.http-method=POST",
"http.request.processor.http-method-expression=headers['method']")
.run(context -> {
assertThatIllegalStateException().isThrownBy(() -> {
context.start();
});
});
}
@Test
void cannotSpecifyBothHeadersAndHeadersExpression() {
applicationContextRunner
.withPropertyValues("http.request.processor.headers[Content-Type]=application/json",
"http.request.processor.headers-expression={'Content-Type': headers['content']}")
.run(context -> {
assertThatIllegalStateException().isThrownBy(() -> {
context.start();
});
});
}
@SpringBootApplication
static class HttpRequestProcessorApp {
}
}

View File

@@ -0,0 +1,2 @@
integration:\
org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration