Adding script-processor apps

https://github.com/spring-cloud/stream-applications/issues/34

* Addressing PR review comments

applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorConfiguration.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2016-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.stream.app.processor.script;
+
+import java.util.function.Function;
+import java.util.regex.Matcher;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.core.io.ByteArrayResource;
+import org.springframework.core.io.Resource;
+import org.springframework.integration.handler.MessageProcessor;
+import org.springframework.integration.scripting.ScriptVariableGenerator;
+import org.springframework.integration.scripting.dsl.Scripts;
+import org.springframework.messaging.Message;
+
+/**
+ * A Processor module that transforms messages using a supplied script. The script
+ * code is passed in directly via property. For more information on Spring script
+ * processing, see
+ * <a href=
+ * "https://spring.io/blog/2011/12/08/spring-integration-scripting-support-part-1">
+ * this blog article</a>.
+ *
+ * @author Andy Clement
+ * @author Gary Russell
+ * @author Chris Schaefer
+ * @author Artme Bilan
+ * @author Soby Chacko
+ */
+@Configuration
+@EnableConfigurationProperties(ScriptProcessorProperties.class)
+@Import(ScriptVariableGeneratorConfiguration.class)
+public class ScriptProcessorConfiguration {
+
+	private static final String NEWLINE_ESCAPE = Matcher.quoteReplacement("\\n");
+
+	private static final String DOUBLE_DOUBLE_QUOTE = Matcher.quoteReplacement("\"\"");
+
+	private static final Log logger = LogFactory.getLog(ScriptProcessorConfiguration.class);
+
+	@Autowired
+	private ScriptProcessorProperties properties;
+
+	@Autowired
+	private ScriptVariableGenerator scriptVariableGenerator;
+
+	@Bean
+	public Function<Message<?>, Object> scriptProcessorFunction() {
+		return processor()::processMessage;
+	}
+
+	@Bean
+	public MessageProcessor<?> processor() {
+		String language = this.properties.getLanguage();
+		String script = this.properties.getScript();
+		logger.info(String.format("Input script is '%s', language is '%s'", script, language));
+		Resource scriptResource = new ByteArrayResource(decodeScript(script).getBytes());
+
+		return Scripts.processor(scriptResource)
+				.lang(language)
+				.variableGenerator(scriptVariableGenerator)
+				.get();
+	}
+
+	private static String decodeScript(String script) {
+		String toProcess = script;
+		// If it has both a leading and trailing double quote, remove them
+		if (toProcess.startsWith("\"") && toProcess.endsWith("\"")) {
+			toProcess = script.substring(1, script.length() - 1);
+		}
+		return toProcess.replaceAll(NEWLINE_ESCAPE, "\n").replaceAll(DOUBLE_DOUBLE_QUOTE, "\"");
+	}
+}

applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorProperties.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2016-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.stream.app.processor.script;
+
+import java.util.Properties;
+
+import javax.validation.constraints.NotNull;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.core.io.Resource;
+import org.springframework.validation.annotation.Validated;
+
+/**
+ * Configuration properties for the Scriptable Transform Processor module.
+ *
+ * @author Andy Clement
+ */
+@ConfigurationProperties("script-processor")
+@Validated
+public class ScriptProcessorProperties {
+
+	/**
+	 * Language of the text in the script property. Supported: groovy, javascript, ruby, python.
+	 */
+	@NotNull
+	private String language;
+
+	/*
+	 * Extra notes on the script parameter. The UI will typically look after encoding
+	 * newlines and double quotes when packaging the value to pass to the script
+	 * property. If not using the UI, attempting to define
+	 * a script directly in the shell for example, it is important to note:
+	 * - newlines should be escaped (\\n)
+	 * - a single " should be expressed in a pair "" - the DSL parser recognizes this pattern
+	 * - If the script starts and ends with a " then they will be stripped off before treating what is
+	 *   left as the script.
+	 *
+	 * Examples:
+	 * ruby: --script="return ""#{payload.upcase}"""
+	 * javascript: --script="function double(a) {\\n return a+"" + ""+a;\\n}\\ndouble(payload);"
+	 */
+	/**
+	 * Text of the script.
+	 */
+	@NotNull
+	private String script;
+
+	/**
+	 * Variable bindings as a new line delimited string of name-value pairs, e.g. 'foo=bar\n baz=car'.
+	 */
+	private Properties variables;
+
+	/**
+	 * The location of a properties file containing custom script variable bindings.
+	 */
+	private Resource variablesLocation;
+
+
+	public String getLanguage() {
+		return this.language;
+	}
+
+	public void setLanguage(String language) {
+		this.language = language;
+	}
+
+	public String getScript() {
+		return this.script;
+	}
+
+	public void setScript(String script) {
+		this.script = script;
+	}
+
+	public Properties getVariables() {
+		return variables;
+	}
+
+	public void setVariables(Properties variables) {
+		this.variables = variables;
+	}
+
+	public Resource getVariablesLocation() {
+		return variablesLocation;
+	}
+
+	public void setVariablesLocation(Resource variablesLocation) {
+		this.variablesLocation = variablesLocation;
+	}
+}

applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptVariableGeneratorConfiguration.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2015-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.stream.app.processor.script;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.io.support.PropertiesLoaderUtils;
+import org.springframework.integration.scripting.DefaultScriptVariableGenerator;
+import org.springframework.integration.scripting.ScriptVariableGenerator;
+import org.springframework.util.CollectionUtils;
+
+/**
+ * Configuration that provides a {@link ScriptVariableGenerator} to customize a script.
+ *
+ * @author David Turanski
+ * @author Eric Bottard
+ * @author Mark Fisher
+ */
+@Configuration
+public class ScriptVariableGeneratorConfiguration {
+
+	@Autowired
+	private ScriptProcessorProperties properties;
+
+	@Bean(name = "variableGenerator")
+	public ScriptVariableGenerator scriptVariableGenerator() throws IOException {
+		Map<String, Object> variables = new HashMap<>();
+		CollectionUtils.mergePropertiesIntoMap(properties.getVariables(), variables);
+		if (properties.getVariablesLocation() != null) {
+			CollectionUtils.mergePropertiesIntoMap(
+					PropertiesLoaderUtils.loadProperties(properties.getVariablesLocation()), variables);
+		}
+		return new DefaultScriptVariableGenerator(variables);
+	}
+}

applications/processor/script-processor/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties

@@ -0,0 +1,4 @@
+configuration-properties.classes=org.springframework.cloud.stream.app.processor.script.ScriptProcessorProperties
+
+
+

applications/processor/script-processor/src/test/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorIntegrationTests.java

@@ -0,0 +1,240 @@
+/*
+ * Copyright 2015-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.stream.app.processor.script;
+
+import java.io.IOException;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+
+import org.springframework.boot.WebApplicationType;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.cloud.stream.binder.test.InputDestination;
+import org.springframework.cloud.stream.binder.test.OutputDestination;
+import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.annotation.Import;
+import org.springframework.messaging.Message;
+import org.springframework.messaging.support.GenericMessage;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Integration Tests for the Script Processor.
+ *
+ * @author Andy Clement
+ * @author Artem Bilan
+ * @author Gary Russell
+ * @author Chris Schaefer
+ * @author Soby Chacko
+ */
+public class ScriptProcessorIntegrationTests {
+
+	@Test
+	public void testJavascriptFunctions() throws IOException {
+		try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
+				TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class))
+				.web(WebApplicationType.NONE)
+				.run("--spring.cloud.function.definition=scriptProcessorFunction",
+						"--script-processor.script=function add(a,b) { return a+b;}; add(1,3)",
+						"--script-processor.language=js")) {
+
+			InputDestination processorInput = context.getBean(InputDestination.class);
+			OutputDestination processorOutput = context.getBean(OutputDestination.class);
+
+			processorInput.send(new GenericMessage<>("hello world"));
+			Message<byte[]> sourceMessage = processorOutput.receive(10000);
+			assertThat(new String(sourceMessage.getPayload())).matches(s -> s.equals("4") || s.equals("4.0"));
+			ObjectMapper objectMapper = new ObjectMapper();
+			final Integer deserializedValue = objectMapper.readValue(sourceMessage.getPayload(), Integer.class);
+			assertThat(deserializedValue).matches(i -> i == 4 || i == 4.0);
+		}
+	}
+
+	@Test
+	public void testJavascriptVariableTake1() {
+		try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
+				TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class))
+				.web(WebApplicationType.NONE)
+				.run("--spring.cloud.function.definition=scriptProcessorFunction",
+						"--script-processor.variables=foo=\\\40WORLD",
+						"--script-processor.script=payload+foo",
+						"--script-processor.language=js")) {
+
+			InputDestination processorInput = context.getBean(InputDestination.class);
+			OutputDestination processorOutput = context.getBean(OutputDestination.class);
+
+			processorInput.send(new GenericMessage<>("hello world"));
+			Message<byte[]> sourceMessage = processorOutput.receive(10000);
+			assertThat(new String(sourceMessage.getPayload())).isEqualTo("hello world WORLD");
+		}
+	}
+
+	@Test
+	public void testJavascriptVariableTake2() {
+		try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
+				TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class))
+				.web(WebApplicationType.NONE)
+				.run("--spring.cloud.function.definition=scriptProcessorFunction",
+						"--script-processor.variables=limit=5",
+						"--script-processor.script=payload*limit",
+						"--script-processor.language=js")) {
+
+			InputDestination processorInput = context.getBean(InputDestination.class);
+			OutputDestination processorOutput = context.getBean(OutputDestination.class);
+
+			processorInput.send(new GenericMessage<>(9));
+			Message<byte[]> sourceMessage = processorOutput.receive(10000);
+			assertThat(new String(sourceMessage.getPayload())).isEqualTo("45.0");
+		}
+	}
+
+	@Test
+	public void testGroovyBasic() {
+		try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
+				TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class))
+				.web(WebApplicationType.NONE)
+				.run("--spring.cloud.function.definition=scriptProcessorFunction",
+						"--script-processor.script=payload+foo",
+						"--script-processor.language=groovy",
+						"--script-processor.variables=foo=\\\40WORLD")) {
+
+			InputDestination processorInput = context.getBean(InputDestination.class);
+			OutputDestination processorOutput = context.getBean(OutputDestination.class);
+
+			processorInput.send(new GenericMessage<>("hello world"));
+			Message<byte[]> sourceMessage = processorOutput.receive(10000);
+			assertThat(new String(sourceMessage.getPayload())).isEqualTo("hello world WORLD");
+		}
+	}
+
+	@Test
+	public void testGroovyComplex() {
+		try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
+				TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class))
+				.web(WebApplicationType.NONE)
+				.run("--spring.cloud.function.definition=scriptProcessorFunction",
+						"--script-processor.script=payload.substring(0, limit as int) + foo",
+						"--script-processor.language=groovy",
+						"--script-processor.variables=limit=5 \n foo=\\\40WORLD")) {
+
+			InputDestination processorInput = context.getBean(InputDestination.class);
+			OutputDestination processorOutput = context.getBean(OutputDestination.class);
+
+			processorInput.send(new GenericMessage<>("hello world"));
+			Message<byte[]> sourceMessage = processorOutput.receive(10000);
+			assertThat(new String(sourceMessage.getPayload())).isEqualTo("hello WORLD");
+		}
+	}
+
+	@Test
+	public void testRubyScript() {
+		try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
+				TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class))
+				.web(WebApplicationType.NONE)
+				.run("--spring.cloud.function.definition=scriptProcessorFunction",
+						"--script-processor.script=return \"\"#{payload.upcase}\"\"",
+						"--script-processor.language=ruby")) {
+
+			InputDestination processorInput = context.getBean(InputDestination.class);
+			OutputDestination processorOutput = context.getBean(OutputDestination.class);
+
+			processorInput.send(new GenericMessage<>("hello world"));
+			Message<byte[]> sourceMessage = processorOutput.receive(10000);
+			assertThat(new String(sourceMessage.getPayload())).isEqualTo("HELLO WORLD");
+		}
+	}
+
+	@Test
+	public void testRubyScriptComplex() {
+		try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
+				TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class))
+				.web(WebApplicationType.NONE)
+				.run("--spring.cloud.function.definition=scriptProcessorFunction",
+						"--script-processor.script=\"def foo(x)\\n  return x+5\\nend\\nfoo(payload)\\n\"",
+						"--script-processor.language=ruby")) {
+
+			InputDestination processorInput = context.getBean(InputDestination.class);
+			OutputDestination processorOutput = context.getBean(OutputDestination.class);
+
+			processorInput.send(new GenericMessage<>(9));
+			Message<byte[]> sourceMessage = processorOutput.receive(10000);
+			assertThat(new String(sourceMessage.getPayload())).isEqualTo("14");
+		}
+	}
+
+	@Test
+	public void testPython() {
+		try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
+				TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class))
+				.web(WebApplicationType.NONE)
+				.run("--spring.cloud.function.definition=scriptProcessorFunction",
+						"--script-processor.script=\"def multiply(x,y):\\n  return x*y\\nanswer = multiply(payload,5)\\n\"",
+						"--script-processor.language=python")) {
+
+			InputDestination processorInput = context.getBean(InputDestination.class);
+			OutputDestination processorOutput = context.getBean(OutputDestination.class);
+
+			processorInput.send(new GenericMessage<>(6));
+			Message<byte[]> sourceMessage = processorOutput.receive(10000);
+			assertThat(new String(sourceMessage.getPayload())).isEqualTo("30");
+		}
+	}
+
+	@Test
+	public void testPythonComplex() {
+		try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
+				TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class))
+				.web(WebApplicationType.NONE)
+				.run("--spring.cloud.function.definition=scriptProcessorFunction",
+						"--script-processor.script=\"def concat(x,y):\\n  return x+y\\nanswer = concat(\"\"hello \"\",payload)\\n\"",
+						"--script-processor.language=python")) {
+
+			InputDestination processorInput = context.getBean(InputDestination.class);
+			OutputDestination processorOutput = context.getBean(OutputDestination.class);
+
+			processorInput.send(new GenericMessage<>("world"));
+			Message<byte[]> sourceMessage = processorOutput.receive(10000);
+			assertThat(new String(sourceMessage.getPayload())).isEqualTo("hello world");
+		}
+	}
+
+	@Test
+	public void testGroovyToJavascript() {
+		try (ConfigurableApplicationContext context = new SpringApplicationBuilder(
+				TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class))
+				.web(WebApplicationType.NONE)
+				.run("--spring.cloud.function.definition=scriptProcessorFunction",
+						"--script-processor.script=\"@Grab('org.grooscript:grooscript:1.3.0')\\n import org.grooscript.GrooScript\\n GrooScript.convert(new String(payload))\\n\"",
+						"--script-processor.language=groovy")) {
+
+			InputDestination processorInput = context.getBean(InputDestination.class);
+			OutputDestination processorOutput = context.getBean(OutputDestination.class);
+
+			processorInput.send(new GenericMessage<>("def age=18"));
+			Message<byte[]> sourceMessage = processorOutput.receive(10000);
+			assertThat(new String(sourceMessage.getPayload())).isEqualTo("var age = 18;"
+					+ System.getProperty("line.separator"));
+		}
+	}
+
+	@EnableAutoConfiguration
+	@Import({ScriptProcessorConfiguration.class})
+	public static class ScriptProcessorTestConfiguration {
+	}
+}

applications/processor/script-processor/src/test/resources/script-with-grab.groovy

@@ -0,0 +1,6 @@
+@Grab('org.grooscript:grooscript:1.3.0')
+import org.grooscript.GrooScript
+@Grab('org.grooscript:grooscript:1.3.0')
+import org.grooscript.GrooScript
+
+GrooScript.convert(new String(payload))

applications/processor/script-processor/src/test/resources/script.groovy

@@ -0,0 +1 @@
+payload.substring(0, limit as int) + foo