Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-9507: Migrate Python support to GraalVM Polyglot

Browse Source 
Fixes: #9507
Issue link: https://github.com/spring-projects/spring-integration/issues/9507

* Deprecate `PythonScriptExecutor` in favor of `PolyglotScriptExecutor` with a `python` as language
* Add handling for `PolyglotWrapper` return type of the script evaluation
* Rework `DeriveLanguageFromExtensionTests.testParseLanguage()` to the `@ParameterizedTest`
This commit is contained in:
Artem Bilan
2024-09-24 17:19:43 -04:00
parent 8082f3c3f9
commit ec31a5bed8
12 changed files with 101 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
build.gradle
View File

@@ -87,7 +87,6 @@ ext {
jsonpathVersion = '2.9.0'
junit4Version = '4.13.2'
junitJupiterVersion = '5.11.0'
jythonVersion = '2.7.4'
kotlinCoroutinesVersion = '1.8.1'
kryoVersion = '5.6.0'
lettuceVersion = '6.4.0.RELEASE'
@@ -866,17 +865,18 @@ project('spring-integration-scripting') {
optionalApi 'org.jetbrains.kotlin:kotlin-scripting-jsr223'
provided "org.graalvm.sdk:graal-sdk:$graalvmVersion"
provided "org.graalvm.polyglot:js:$graalvmVersion"
provided "org.graalvm.polyglot:python:$graalvmVersion"
testImplementation "org.jruby:jruby-complete:$jrubyVersion"
testImplementation 'org.apache.groovy:groovy-jsr223'
testImplementation "org.python:jython-standalone:$jythonVersion"
}
tasks.withType(JavaForkOptions) {
jvmArgs '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED',
'--add-opens', 'java.base/java.io=ALL-UNNAMED',
'--add-opens', 'java.base/java.lang=ALL-UNNAMED',
'--add-opens', 'java.base/java.util=ALL-UNNAMED'
'--add-opens', 'java.base/java.util=ALL-UNNAMED',
'-Dpolyglot.engine.WarnInterpreterOnly=false'
}
}

1
spring-integration-core/src/main/java/org/springframework/integration/handler/advice/CacheRequestHandlerAdvice.java
View File

@@ -21,7 +21,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.beans.factory.SmartInitializingSingleton;

25
spring-integration-scripting/src/main/java/org/springframework/integration/scripting/PolyglotScriptExecutor.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2022 the original author or authors.
* Copyright 2022-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
package org.springframework.integration.scripting;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.graalvm.polyglot.Context;
@@ -36,7 +38,7 @@ public class PolyglotScriptExecutor implements ScriptExecutor {
private final String language;
private Context.Builder contextBuilder;
private final Context.Builder contextBuilder;
/**
* Construct an executor based on the provided language id.
@@ -67,11 +69,28 @@ public class PolyglotScriptExecutor implements ScriptExecutor {
Value bindings = context.getBindings(this.language);
variables.forEach(bindings::putMember);
}
return context.eval(this.language, scriptSource.getScriptAsString()).as(Object.class);
String scriptAsString = scriptSource.getScriptAsString();
Object result = context.eval(this.language, scriptAsString).as(Object.class);
// We have to copy all the expected PolyglotWrapper instances before context is closed.
if (result instanceof Map<?, ?> map) {
String returnVariable = parseReturnVariable(scriptAsString);
result = map.get(returnVariable);
}
if (result instanceof List<?> list) {
result = new ArrayList<>(list);
}
return result;
}
catch (Exception ex) {
throw new ScriptingException(ex.getMessage(), ex);
}
}
private static String parseReturnVariable(String script) {
String[] lines = script.trim().split("\n");
String lastLine = lines[lines.length - 1];
String[] tokens = lastLine.split("=");
return tokens[0].trim();
}
}

5
spring-integration-scripting/src/main/java/org/springframework/integration/scripting/jsr223/PythonScriptExecutor.java
View File

@@ -32,9 +32,14 @@ import javax.script.ScriptEngine;
*
* @author David Turanski
* @author Gary Russell
* @author Artem Bilan
*
* @since 2.1
*
* @deprecated in favor of {@link org.springframework.integration.scripting.PolyglotScriptExecutor}
* with a {@code python} language argument.
*/
@Deprecated(forRemoval = true, since = "6.4")
public class PythonScriptExecutor extends AbstractScriptExecutor {
public PythonScriptExecutor() {

19
spring-integration-scripting/src/main/java/org/springframework/integration/scripting/jsr223/ScriptExecutorFactory.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -35,7 +35,7 @@ public final class ScriptExecutorFactory {
public static ScriptExecutor getScriptExecutor(String language) {
if (language.equalsIgnoreCase("python") || language.equalsIgnoreCase("jython")) {
return new PythonScriptExecutor();
return new PolyglotScriptExecutor("python");
}
else if (language.equalsIgnoreCase("ruby") || language.equalsIgnoreCase("jruby")) {
return new RubyScriptExecutor();
@@ -56,11 +56,16 @@ public final class ScriptExecutorFactory {
int index = scriptLocation.lastIndexOf('.') + 1;
Assert.state(index > 0, () -> "Unable to determine language for script '" + scriptLocation + "'");
String extension = scriptLocation.substring(index);
if (extension.equals("kts")) {
return "kotlin";
}
else if (extension.equals("js")) {
return "js";
switch (extension) {
case "kts" -> {
return "kotlin";
}
case "js" -> {
return "js";
}
case "py" -> {
return "python";
}
}
ScriptEngineManager engineManager = new ScriptEngineManager();
ScriptEngine engine = engineManager.getEngineByExtension(extension);

9
spring-integration-scripting/src/test/java/org/springframework/integration/scripting/jsr223/DeriveLanguageFromExtensionTests-context.xml
View File

@@ -5,8 +5,9 @@
xsi:schemaLocation="http://www.springframework.org/schema/beans https://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration/scripting https://www.springframework.org/schema/integration/scripting/spring-integration-scripting.xsd">
<int-script:script location="foo.rb"/>
<int-script:script location="foo.groovy"/>
<int-script:script location="foo.py"/>
<int-script:script location="foo.kts"/>
<int-script:script location="script.rb"/>
<int-script:script location="script.groovy"/>
<int-script:script location="script.py"/>
<int-script:script location="script.kts"/>
<int-script:script location="script.js"/>
</beans>

57
spring-integration-scripting/src/test/java/org/springframework/integration/scripting/jsr223/DeriveLanguageFromExtensionTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,14 +16,20 @@
package org.springframework.integration.scripting.jsr223;
import java.util.Map;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.aggregator.ArgumentsAccessor;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.springframework.beans.factory.BeanDefinitionStoreException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.integration.scripting.PolyglotScriptExecutor;
import org.springframework.integration.scripting.ScriptExecutor;
import org.springframework.integration.test.util.TestUtils;
import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
@@ -41,30 +47,26 @@ public class DeriveLanguageFromExtensionTests {
@Autowired
private ApplicationContext ctx;
@Test
public void testParseLanguage() {
String[] langs = {"ruby", "Groovy", "python", "kotlin"};
Class<?>[] executors = {
RubyScriptExecutor.class,
DefaultScriptExecutor.class,
PythonScriptExecutor.class,
DefaultScriptExecutor.class
};
@ParameterizedTest
@MethodSource("languageExecutorSource")
public void testParseLanguage(String language, Class<?> executorClass, ArgumentsAccessor argumentsAccessor) {
assertThat(this.ctx.getBeansOfType(ScriptExecutingMessageProcessor.class)).hasSize(5);
Map<String, ScriptExecutingMessageProcessor> scriptProcessors =
this.ctx.getBeansOfType(ScriptExecutingMessageProcessor.class);
assertThat(scriptProcessors.size()).isEqualTo(4);
var processor =
ctx.getBean(
"org.springframework.integration.scripting.jsr223.ScriptExecutingMessageProcessor#" +
(argumentsAccessor.getInvocationIndex() - 1),
ScriptExecutingMessageProcessor.class);
for (int i = 0; i < 4; i++) {
ScriptExecutingMessageProcessor processor = ctx.getBean(
"org.springframework.integration.scripting.jsr223.ScriptExecutingMessageProcessor#" + i,
ScriptExecutingMessageProcessor.class);
AbstractScriptExecutor executor =
TestUtils.getPropertyValue(processor, "scriptExecutor", AbstractScriptExecutor.class);
assertThat(executor.getScriptEngine().getFactory().getLanguageName()).isEqualTo(langs[i]);
assertThat(executor.getClass()).isEqualTo(executors[i]);
ScriptExecutor executor = TestUtils.getPropertyValue(processor, "scriptExecutor", ScriptExecutor.class);
if (executor instanceof PolyglotScriptExecutor) {
assertThat(TestUtils.getPropertyValue(executor, "language")).isEqualTo(language);
}
else {
AbstractScriptExecutor abstractScriptExecutor = (AbstractScriptExecutor) executor;
assertThat(abstractScriptExecutor.getScriptEngine().getFactory().getLanguageName()).isEqualTo(language);
}
assertThat(executor.getClass()).isEqualTo(executorClass);
}
@Test
@@ -85,4 +87,13 @@ public class DeriveLanguageFromExtensionTests {
.withStackTraceContaining("Unable to determine language for script 'foo'");
}
private static Stream<Arguments> languageExecutorSource() {
return Stream.of(
Arguments.of("ruby", RubyScriptExecutor.class),
Arguments.of("Groovy", DefaultScriptExecutor.class),
Arguments.of("python", PolyglotScriptExecutor.class),
Arguments.of("kotlin", DefaultScriptExecutor.class),
Arguments.of("js", PolyglotScriptExecutor.class));
}
}

14
spring-integration-scripting/src/test/java/org/springframework/integration/scripting/jsr223/PythonScriptExecutorTests.java
View File

@@ -22,9 +22,9 @@ import java.util.Map;
import org.assertj.core.api.InstanceOfAssertFactories;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.python.core.PyTuple;
import org.springframework.core.io.ClassPathResource;
import org.springframework.integration.scripting.PolyglotScriptExecutor;
import org.springframework.integration.scripting.ScriptExecutor;
import org.springframework.scripting.ScriptSource;
import org.springframework.scripting.support.ResourceScriptSource;
@@ -44,7 +44,7 @@ public class PythonScriptExecutorTests {
@BeforeEach
public void init() {
this.executor = new PythonScriptExecutor();
this.executor = new PolyglotScriptExecutor("python");
}
@Test
@@ -76,11 +76,8 @@ public class PythonScriptExecutorTests {
new ClassPathResource("/org/springframework/integration/scripting/jsr223/test3.py"));
Object obj = this.executor.executeScript(source);
assertThat(obj)
.isNotNull()
.isInstanceOf(PyTuple.class)
.asInstanceOf(InstanceOfAssertFactories.LIST)
.element(0)
.isEqualTo(1);
.containsOnly(1, 2, 3);
}
@Test
@@ -92,11 +89,8 @@ public class PythonScriptExecutorTests {
variables.put("foo", "bar");
Object obj = this.executor.executeScript(source, variables);
assertThat(obj)
.isNotNull()
.isInstanceOf(PyTuple.class)
.asInstanceOf(InstanceOfAssertFactories.LIST)
.element(0)
.isEqualTo(1);
.containsOnly(1, 2, 3);
}
@Test

54
spring-integration-scripting/src/test/java/org/springframework/integration/scripting/jsr223/PythonVariableParserTests.java
View File

@@ -1,54 +0,0 @@
/*
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.scripting.jsr223;
import java.io.IOException;
import org.junit.Test;
import org.springframework.core.io.ClassPathResource;
import org.springframework.scripting.ScriptSource;
import org.springframework.scripting.support.ResourceScriptSource;
import static org.assertj.core.api.Assertions.assertThat;
/**
* @author David Turanski
*
*/
public class PythonVariableParserTests {
@Test
public void testBasic() throws IOException {
String var = PythonScriptExecutor.PythonVariableParser.parseReturnVariable("x=2");
assertThat(var).isEqualTo("x");
var = PythonScriptExecutor.PythonVariableParser.parseReturnVariable("\n\n\nx = 2\n\n\n");
assertThat(var).isEqualTo("x");
var = PythonScriptExecutor.PythonVariableParser.parseReturnVariable("\n\n\nx\n\n\n");
}
@Test
public void test2() throws IOException {
ScriptSource source =
new ResourceScriptSource(new ClassPathResource("/org/springframework/integration/scripting/jsr223/test2.py"));
String var = PythonScriptExecutor.PythonVariableParser.parseReturnVariable(source.getScriptAsString());
assertThat(var).isEqualTo("bar");
}
}

5
spring-integration-scripting/src/test/java/org/springframework/integration/scripting/jsr223/test2.py
View File

@@ -1,5 +0,0 @@
def foo(y):
x=y
return x
bar = foo(7)

15
src/reference/antora/modules/ROOT/pages/scripting.adoc
View File

@@ -28,7 +28,7 @@ compile "org.springframework.integration:spring-integration-scripting:{project-v
----
======
In addition, you need to add a script engine implementation, e.g. JRuby, Jython.
In addition, you need to add a script engine implementation, e.g. JRuby.
Starting with version 5.2, Spring Integration provides a Kotlin Jsr223 support.
You need to add this dependency into your project to make it working:
@@ -293,9 +293,14 @@ They are mutually exclusive.
[[graalmv-polyglot]]
=== GraalVM Polyglot
Starting with version 6.0, the framework provides a `PolyglotScriptExecutor` which is based the https://www.graalvm.org/22.2/reference-manual/embed-languages[GraalVM Polyglot API].
Starting with version 6.0, the framework provides a `PolyglotScriptExecutor` which is based the https://www.graalvm.org/latest/reference-manual/languages/[GraalVM Polyglot API].
The JSR223 engine implementation for JavaScript, removed from Java by itself, has been replaced by using this new script executor.
See more information about enabling JavaScript support in GraalVM and what https://www.graalvm.org/22.2/reference-manual/js[configuration options] can be propagated via script variables.
See more information about enabling JavaScript support in GraalVM and what https://www.graalvm.org/latest/reference-manual/js/[configuration options] can be propagated via script variables.
Starting with version 6.4, the Python scripts support has been migrated to GraalVM Polyglot as well.
Now these scripts can be written in Python 3.x and can use third-party libraries.
See https://www.graalvm.org/latest/reference-manual/python/[GraalPy] documentation for more information.
By default, the framework sets `allowAllAccess` to `true` on the shared Polyglot `Context` which enables this interaction with host JVM:
* The creation and use of new threads.
@@ -307,6 +312,4 @@ By default, the framework sets `allowAllAccess` to `true` on the shared Polyglot
* The creation and use of new sub-processes.
* The access to process environment variables.
This can be customized via overloaded `PolyglotScriptExecutor` constructor which accepts a `org.graalvm.polyglot.Context.Builder`.
To enable this JavaScript support, GraalVM with the `js` component installed has to be used or, when using a regular JVM, the `org.graalvm.sdk:graal-sdk` and `org.graalvm.js:js` dependencies must be included.
This can be customized via overloaded `PolyglotScriptExecutor` constructor which accepts a `org.graalvm.polyglot.Context.Builder`.

7
src/reference/antora/modules/ROOT/pages/whats-new.adoc
View File

@@ -84,3 +84,10 @@ See xref:mqtt.adoc[MQTT Support] for more information.
The `ZipTransformer` now exposes a `fileNameGenerator` property to customize a target zip file (and optional zip entry) name generation.
See xref:zip.adoc[Zip Support] for more information.
[[x6.4-scripting-changes]]
=== Scripting Changes
The Python scripts evaluation is now migrated to the GraalVM Polyglot.
See xref:scripting.adoc[Scripting Support] for more information.