diff --git a/applications/processor/pom.xml b/applications/processor/pom.xml index e758b45b..c42c4dda 100644 --- a/applications/processor/pom.xml +++ b/applications/processor/pom.xml @@ -16,6 +16,7 @@ splitter-processor filter-processor transform-processor + script-processor diff --git a/applications/processor/script-processor/README.adoc b/applications/processor/script-processor/README.adoc new file mode 100644 index 00000000..cbe2d9c3 --- /dev/null +++ b/applications/processor/script-processor/README.adoc @@ -0,0 +1,18 @@ +//tag::ref-doc[] += Script Processor + +Processor that transforms messages using a script. The script body is supplied directly +as a property value. The language of the script can be specified (groovy/javascript/ruby/python). + +== Options + +The **$$script-processor$$** $$processor$$ has the following options: + +//tag::configuration-properties[] +$$script-processor.language$$:: $$Language of the text in the script property. Supported: groovy, javascript, ruby, python.$$ *($$String$$, default: `$$$$`)* +$$script-processor.script$$:: $$Text of the script.$$ *($$String$$, default: `$$$$`)* +$$script-processor.variables$$:: $$Variable bindings as a new line delimited string of name-value pairs, e.g. 'foo=bar\n baz=car'.$$ *($$Properties$$, default: `$$$$`)* +$$script-processor.variables-location$$:: $$The location of a properties file containing custom script variable bindings.$$ *($$Resource$$, default: `$$$$`)* +//end::configuration-properties[] + +//end::ref-doc[] diff --git a/applications/processor/script-processor/pom.xml b/applications/processor/script-processor/pom.xml new file mode 100644 index 00000000..86f7ff50 --- /dev/null +++ b/applications/processor/script-processor/pom.xml @@ -0,0 +1,146 @@ + + + 4.0.0 + script-processor + script-processor + script processor apps + 3.0.0-SNAPSHOT + + + org.springframework.cloud.stream.app + stream-applications-core + 3.0.0-SNAPSHOT + + + + + 9.0.5.0 + 2.7.0 + 2.4.0 + + + + + org.springframework.integration + spring-integration-groovy + + + + org.codehaus.groovy + groovy-json + + + org.codehaus.groovy + groovy-nio + + + org.codehaus.groovy + groovy-templates + + + org.codehaus.groovy + groovy-xml + + + org.codehaus.groovy + groovy-dateutil + + + org.codehaus.groovy + groovy-datetime + + + org.codehaus.groovy + groovy-jaxb + + + org.jruby + jruby-complete + ${jruby-complete.version} + + + org.python + jython-standalone + ${jython-standalone.version} + + + org.apache.ivy + ivy + ${apache-ivy.version} + + + org.springframework.cloud.fn + payload-converter-function + 1.0.0-SNAPSHOT + + + org.springframework.boot + spring-boot-configuration-processor + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + + + + + + + org.springframework.cloud + spring-cloud-app-starter-doc-maven-plugin + + + org.springframework.cloud.stream.app.plugin + spring-cloud-stream-app-maven-plugin + + + script + processor + ${project.version} + + org.springframework.cloud.stream.app.processor.script.ScriptProcessorConfiguration.class + + byteArrayTextToString|scriptProcessorFunction + + + + + org.springframework.cloud.stream.app + script-processor + ${project.version} + + + + + + + + + + + true + + spring-snapshots + Spring Snapshots + https://repo.spring.io/libs-snapshot-local + + + + false + + spring-milestones + Spring Milestones + https://repo.spring.io/libs-milestone-local + + + diff --git a/applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorConfiguration.java b/applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorConfiguration.java new file mode 100644 index 00000000..e826c72c --- /dev/null +++ b/applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorConfiguration.java @@ -0,0 +1,94 @@ +/* + * Copyright 2016-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.processor.script; + +import java.util.function.Function; +import java.util.regex.Matcher; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.Resource; +import org.springframework.integration.handler.MessageProcessor; +import org.springframework.integration.scripting.ScriptVariableGenerator; +import org.springframework.integration.scripting.dsl.Scripts; +import org.springframework.messaging.Message; + +/** + * A Processor module that transforms messages using a supplied script. The script + * code is passed in directly via property. For more information on Spring script + * processing, see + * + * this blog article. + * + * @author Andy Clement + * @author Gary Russell + * @author Chris Schaefer + * @author Artme Bilan + * @author Soby Chacko + */ +@Configuration +@EnableConfigurationProperties(ScriptProcessorProperties.class) +@Import(ScriptVariableGeneratorConfiguration.class) +public class ScriptProcessorConfiguration { + + private static final String NEWLINE_ESCAPE = Matcher.quoteReplacement("\\n"); + + private static final String DOUBLE_DOUBLE_QUOTE = Matcher.quoteReplacement("\"\""); + + private static final Log logger = LogFactory.getLog(ScriptProcessorConfiguration.class); + + @Autowired + private ScriptProcessorProperties properties; + + @Autowired + private ScriptVariableGenerator scriptVariableGenerator; + + @Bean + public Function, Object> scriptProcessorFunction() { + return processor()::processMessage; + } + + @Bean + public MessageProcessor processor() { + String language = this.properties.getLanguage(); + String script = this.properties.getScript(); + logger.info(String.format("Input script is '%s', language is '%s'", script, language)); + Resource scriptResource = new ByteArrayResource(decodeScript(script).getBytes()); + + return Scripts.processor(scriptResource) + .lang(language) + .variableGenerator(scriptVariableGenerator) + .get(); + } + + private static String decodeScript(String script) { + String toProcess = script; + // If it has both a leading and trailing double quote, remove them + if (toProcess.startsWith("\"") && toProcess.endsWith("\"")) { + toProcess = script.substring(1, script.length() - 1); + } + return toProcess.replaceAll(NEWLINE_ESCAPE, "\n").replaceAll(DOUBLE_DOUBLE_QUOTE, "\""); + } +} diff --git a/applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorProperties.java b/applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorProperties.java new file mode 100644 index 00000000..06d39185 --- /dev/null +++ b/applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorProperties.java @@ -0,0 +1,104 @@ +/* + * Copyright 2016-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.processor.script; + +import java.util.Properties; + +import javax.validation.constraints.NotNull; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.core.io.Resource; +import org.springframework.validation.annotation.Validated; + +/** + * Configuration properties for the Scriptable Transform Processor module. + * + * @author Andy Clement + */ +@ConfigurationProperties("script-processor") +@Validated +public class ScriptProcessorProperties { + + /** + * Language of the text in the script property. Supported: groovy, javascript, ruby, python. + */ + @NotNull + private String language; + + /* + * Extra notes on the script parameter. The UI will typically look after encoding + * newlines and double quotes when packaging the value to pass to the script + * property. If not using the UI, attempting to define + * a script directly in the shell for example, it is important to note: + * - newlines should be escaped (\\n) + * - a single " should be expressed in a pair "" - the DSL parser recognizes this pattern + * - If the script starts and ends with a " then they will be stripped off before treating what is + * left as the script. + * + * Examples: + * ruby: --script="return ""#{payload.upcase}""" + * javascript: --script="function double(a) {\\n return a+"" + ""+a;\\n}\\ndouble(payload);" + */ + /** + * Text of the script. + */ + @NotNull + private String script; + + /** + * Variable bindings as a new line delimited string of name-value pairs, e.g. 'foo=bar\n baz=car'. + */ + private Properties variables; + + /** + * The location of a properties file containing custom script variable bindings. + */ + private Resource variablesLocation; + + + public String getLanguage() { + return this.language; + } + + public void setLanguage(String language) { + this.language = language; + } + + public String getScript() { + return this.script; + } + + public void setScript(String script) { + this.script = script; + } + + public Properties getVariables() { + return variables; + } + + public void setVariables(Properties variables) { + this.variables = variables; + } + + public Resource getVariablesLocation() { + return variablesLocation; + } + + public void setVariablesLocation(Resource variablesLocation) { + this.variablesLocation = variablesLocation; + } +} diff --git a/applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptVariableGeneratorConfiguration.java b/applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptVariableGeneratorConfiguration.java new file mode 100644 index 00000000..347e066b --- /dev/null +++ b/applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptVariableGeneratorConfiguration.java @@ -0,0 +1,54 @@ +/* + * Copyright 2015-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.processor.script; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.support.PropertiesLoaderUtils; +import org.springframework.integration.scripting.DefaultScriptVariableGenerator; +import org.springframework.integration.scripting.ScriptVariableGenerator; +import org.springframework.util.CollectionUtils; + +/** + * Configuration that provides a {@link ScriptVariableGenerator} to customize a script. + * + * @author David Turanski + * @author Eric Bottard + * @author Mark Fisher + */ +@Configuration +public class ScriptVariableGeneratorConfiguration { + + @Autowired + private ScriptProcessorProperties properties; + + @Bean(name = "variableGenerator") + public ScriptVariableGenerator scriptVariableGenerator() throws IOException { + Map variables = new HashMap<>(); + CollectionUtils.mergePropertiesIntoMap(properties.getVariables(), variables); + if (properties.getVariablesLocation() != null) { + CollectionUtils.mergePropertiesIntoMap( + PropertiesLoaderUtils.loadProperties(properties.getVariablesLocation()), variables); + } + return new DefaultScriptVariableGenerator(variables); + } +} diff --git a/applications/processor/script-processor/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/processor/script-processor/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties new file mode 100644 index 00000000..bf557032 --- /dev/null +++ b/applications/processor/script-processor/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -0,0 +1,4 @@ +configuration-properties.classes=org.springframework.cloud.stream.app.processor.script.ScriptProcessorProperties + + + diff --git a/applications/processor/script-processor/src/test/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorIntegrationTests.java b/applications/processor/script-processor/src/test/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorIntegrationTests.java new file mode 100644 index 00000000..a702ca31 --- /dev/null +++ b/applications/processor/script-processor/src/test/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorIntegrationTests.java @@ -0,0 +1,240 @@ +/* + * Copyright 2015-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.stream.app.processor.script; + +import java.io.IOException; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import org.springframework.boot.WebApplicationType; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.stream.binder.test.InputDestination; +import org.springframework.cloud.stream.binder.test.OutputDestination; +import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Import; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.GenericMessage; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Integration Tests for the Script Processor. + * + * @author Andy Clement + * @author Artem Bilan + * @author Gary Russell + * @author Chris Schaefer + * @author Soby Chacko + */ +public class ScriptProcessorIntegrationTests { + + @Test + public void testJavascriptFunctions() throws IOException { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=scriptProcessorFunction", + "--script-processor.script=function add(a,b) { return a+b;}; add(1,3)", + "--script-processor.language=js")) { + + InputDestination processorInput = context.getBean(InputDestination.class); + OutputDestination processorOutput = context.getBean(OutputDestination.class); + + processorInput.send(new GenericMessage<>("hello world")); + Message sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload())).matches(s -> s.equals("4") || s.equals("4.0")); + ObjectMapper objectMapper = new ObjectMapper(); + final Integer deserializedValue = objectMapper.readValue(sourceMessage.getPayload(), Integer.class); + assertThat(deserializedValue).matches(i -> i == 4 || i == 4.0); + } + } + + @Test + public void testJavascriptVariableTake1() { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=scriptProcessorFunction", + "--script-processor.variables=foo=\\\40WORLD", + "--script-processor.script=payload+foo", + "--script-processor.language=js")) { + + InputDestination processorInput = context.getBean(InputDestination.class); + OutputDestination processorOutput = context.getBean(OutputDestination.class); + + processorInput.send(new GenericMessage<>("hello world")); + Message sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload())).isEqualTo("hello world WORLD"); + } + } + + @Test + public void testJavascriptVariableTake2() { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=scriptProcessorFunction", + "--script-processor.variables=limit=5", + "--script-processor.script=payload*limit", + "--script-processor.language=js")) { + + InputDestination processorInput = context.getBean(InputDestination.class); + OutputDestination processorOutput = context.getBean(OutputDestination.class); + + processorInput.send(new GenericMessage<>(9)); + Message sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload())).isEqualTo("45.0"); + } + } + + @Test + public void testGroovyBasic() { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=scriptProcessorFunction", + "--script-processor.script=payload+foo", + "--script-processor.language=groovy", + "--script-processor.variables=foo=\\\40WORLD")) { + + InputDestination processorInput = context.getBean(InputDestination.class); + OutputDestination processorOutput = context.getBean(OutputDestination.class); + + processorInput.send(new GenericMessage<>("hello world")); + Message sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload())).isEqualTo("hello world WORLD"); + } + } + + @Test + public void testGroovyComplex() { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=scriptProcessorFunction", + "--script-processor.script=payload.substring(0, limit as int) + foo", + "--script-processor.language=groovy", + "--script-processor.variables=limit=5 \n foo=\\\40WORLD")) { + + InputDestination processorInput = context.getBean(InputDestination.class); + OutputDestination processorOutput = context.getBean(OutputDestination.class); + + processorInput.send(new GenericMessage<>("hello world")); + Message sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload())).isEqualTo("hello WORLD"); + } + } + + @Test + public void testRubyScript() { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=scriptProcessorFunction", + "--script-processor.script=return \"\"#{payload.upcase}\"\"", + "--script-processor.language=ruby")) { + + InputDestination processorInput = context.getBean(InputDestination.class); + OutputDestination processorOutput = context.getBean(OutputDestination.class); + + processorInput.send(new GenericMessage<>("hello world")); + Message sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload())).isEqualTo("HELLO WORLD"); + } + } + + @Test + public void testRubyScriptComplex() { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=scriptProcessorFunction", + "--script-processor.script=\"def foo(x)\\n return x+5\\nend\\nfoo(payload)\\n\"", + "--script-processor.language=ruby")) { + + InputDestination processorInput = context.getBean(InputDestination.class); + OutputDestination processorOutput = context.getBean(OutputDestination.class); + + processorInput.send(new GenericMessage<>(9)); + Message sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload())).isEqualTo("14"); + } + } + + @Test + public void testPython() { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=scriptProcessorFunction", + "--script-processor.script=\"def multiply(x,y):\\n return x*y\\nanswer = multiply(payload,5)\\n\"", + "--script-processor.language=python")) { + + InputDestination processorInput = context.getBean(InputDestination.class); + OutputDestination processorOutput = context.getBean(OutputDestination.class); + + processorInput.send(new GenericMessage<>(6)); + Message sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload())).isEqualTo("30"); + } + } + + @Test + public void testPythonComplex() { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=scriptProcessorFunction", + "--script-processor.script=\"def concat(x,y):\\n return x+y\\nanswer = concat(\"\"hello \"\",payload)\\n\"", + "--script-processor.language=python")) { + + InputDestination processorInput = context.getBean(InputDestination.class); + OutputDestination processorOutput = context.getBean(OutputDestination.class); + + processorInput.send(new GenericMessage<>("world")); + Message sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload())).isEqualTo("hello world"); + } + } + + @Test + public void testGroovyToJavascript() { + try (ConfigurableApplicationContext context = new SpringApplicationBuilder( + TestChannelBinderConfiguration.getCompleteConfiguration(ScriptProcessorTestConfiguration.class)) + .web(WebApplicationType.NONE) + .run("--spring.cloud.function.definition=scriptProcessorFunction", + "--script-processor.script=\"@Grab('org.grooscript:grooscript:1.3.0')\\n import org.grooscript.GrooScript\\n GrooScript.convert(new String(payload))\\n\"", + "--script-processor.language=groovy")) { + + InputDestination processorInput = context.getBean(InputDestination.class); + OutputDestination processorOutput = context.getBean(OutputDestination.class); + + processorInput.send(new GenericMessage<>("def age=18")); + Message sourceMessage = processorOutput.receive(10000); + assertThat(new String(sourceMessage.getPayload())).isEqualTo("var age = 18;" + + System.getProperty("line.separator")); + } + } + + @EnableAutoConfiguration + @Import({ScriptProcessorConfiguration.class}) + public static class ScriptProcessorTestConfiguration { + } +} diff --git a/applications/processor/script-processor/src/test/resources/script-with-grab.groovy b/applications/processor/script-processor/src/test/resources/script-with-grab.groovy new file mode 100644 index 00000000..76002c08 --- /dev/null +++ b/applications/processor/script-processor/src/test/resources/script-with-grab.groovy @@ -0,0 +1,6 @@ +@Grab('org.grooscript:grooscript:1.3.0') +import org.grooscript.GrooScript +@Grab('org.grooscript:grooscript:1.3.0') +import org.grooscript.GrooScript + +GrooScript.convert(new String(payload)) diff --git a/applications/processor/script-processor/src/test/resources/script.groovy b/applications/processor/script-processor/src/test/resources/script.groovy new file mode 100644 index 00000000..089a5c9a --- /dev/null +++ b/applications/processor/script-processor/src/test/resources/script.groovy @@ -0,0 +1 @@ +payload.substring(0, limit as int) + foo