From aaa6b6526ca8b7d8649a33f49796246ab1640c32 Mon Sep 17 00:00:00 2001 From: Andy Clement Date: Sun, 5 Nov 2017 11:26:57 -0800 Subject: [PATCH] Faster compilation of functions This change includes caching and smarter analysis of classpaths to limit the amount of jar walking necessary to find dependencies when compiling. It also includes some new tests that verify packaging of dependencies in boot style form (BOOT-INF/lib and BOOT-INF/classes). --- ...eableFilterableJavaFileObjectIterable.java | 26 +- .../compiler/java/IterableClasspath.java | 22 +- .../java/MemoryBasedJavaFileManager.java | 285 ++++++++++- .../compiler/java/RuntimeJavaCompiler.java | 2 +- .../CompilerDependencyResolutionTests.java | 441 ++++++++++++++++++ 5 files changed, 742 insertions(+), 34 deletions(-) create mode 100644 spring-cloud-function-compiler/src/test/java/org/springframework/cloud/function/compiler/CompilerDependencyResolutionTests.java diff --git a/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/CloseableFilterableJavaFileObjectIterable.java b/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/CloseableFilterableJavaFileObjectIterable.java index 7b89da2d6..5b0aaef9c 100644 --- a/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/CloseableFilterableJavaFileObjectIterable.java +++ b/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/CloseableFilterableJavaFileObjectIterable.java @@ -20,6 +20,8 @@ import java.io.File; import javax.tools.JavaFileObject; +import org.springframework.cloud.function.compiler.java.MemoryBasedJavaFileManager.CompilationInfoCache; + /** * Common superclass for iterables that need to handle closing when finished * with and that need to handle possible constraints on the values that @@ -35,22 +37,26 @@ public abstract class CloseableFilterableJavaFileObjectIterable implements Itera private final static String BOOT_PACKAGING_PREFIX_FOR_CLASSES = "BOOT-INF/classes/"; // If set specifies the package the iterator consumer is interested in. Only - // return results in this package. - private String packageNameFilter; + // return results in this package. Will have a trailing separator to speed + // matching. '/' on its own represents the default package + protected String packageNameFilter; // Indicates whether the consumer of the iterator wants to see classes // that are in subpackages of those matching the filter. - private boolean includeSubpackages; + protected boolean includeSubpackages; + + protected CompilationInfoCache compilationInfoCache; - public CloseableFilterableJavaFileObjectIterable(String packageNameFilter, boolean includeSubpackages) { + public CloseableFilterableJavaFileObjectIterable(CompilationInfoCache compilationInfoCache, String packageNameFilter, boolean includeSubpackages) { if (packageNameFilter!=null && packageNameFilter.contains(File.separator)) { throw new IllegalArgumentException("Package name filters should use dots to separate components: "+packageNameFilter); } + this.compilationInfoCache = compilationInfoCache; // Normalize filter to forward slashes this.packageNameFilter = packageNameFilter==null?null:packageNameFilter.replace('.', '/') + '/'; this.includeSubpackages = includeSubpackages; } - + /** * Used by subclasses to check values against any specified constraints. * @@ -68,6 +74,16 @@ public abstract class CloseableFilterableJavaFileObjectIterable implements Itera boolean accept; // Normalize to forward slashes (some jars are producing paths with forward slashes, some with backward slashes) name = name.replace('\\', '/'); + if (packageNameFilter.length() == 1 && packageNameFilter.equals("/")) { + // This is the 'default package' filter representation + if (name.indexOf('/') == -1) { + accept = true; + } else if (BOOT_PACKAGING_AWARE) { + accept = name.startsWith(BOOT_PACKAGING_PREFIX_FOR_CLASSES) && + name.indexOf('/',BOOT_PACKAGING_PREFIX_FOR_CLASSES.length()) == -1; + } + return accept; + } if (includeSubpackages == true) { accept = name.startsWith(packageNameFilter); if (!accept && BOOT_PACKAGING_AWARE) { diff --git a/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/IterableClasspath.java b/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/IterableClasspath.java index 7bd55b688..393bbff80 100644 --- a/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/IterableClasspath.java +++ b/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/IterableClasspath.java @@ -33,6 +33,8 @@ import javax.tools.JavaFileObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.cloud.function.compiler.java.MemoryBasedJavaFileManager.CompilationInfoCache; +import org.springframework.cloud.function.compiler.java.MemoryBasedJavaFileManager.CompilationInfoCache.ArchiveInfo; /** * Iterable that will produce an iterator that returns classes found @@ -46,24 +48,30 @@ public class IterableClasspath extends CloseableFilterableJavaFileObjectIterable private static Logger logger = LoggerFactory.getLogger(IterableClasspath.class); - private static final String BOOT_PACKAGING_PREFIX_FOR_LIBRARIES = "BOOT-INF/lib/"; - private List classpathEntries = new ArrayList<>(); private List openArchives = new ArrayList<>(); /** + * @param compilationInfoCache cache of info that may help accelerate compilation * @param classpath a classpath of jars/directories * @param packageNameFilter an optional package name if choosing to filter (e.g. com.example) * @param includeSubpackages if true, include results in subpackages of the specified package filter */ - IterableClasspath(String classpath, String packageNameFilter, boolean includeSubpackages) { - super(packageNameFilter, includeSubpackages); + IterableClasspath(CompilationInfoCache compilationInfoCache, String classpath, String packageNameFilter, boolean includeSubpackages) { + super(compilationInfoCache, packageNameFilter, includeSubpackages); StringTokenizer tokenizer = new StringTokenizer(classpath, File.pathSeparator); while (tokenizer.hasMoreElements()) { String nextEntry = tokenizer.nextToken(); File f = new File(nextEntry); if (f.exists()) { + // Skip iterating over archives that cannot possibly match the filter + if (this.packageNameFilter != null && this.packageNameFilter.length() > 0) { + ArchiveInfo archiveInfo = compilationInfoCache.getArchiveInfoFor(f); + if (archiveInfo != null && !archiveInfo.containsPackage(this.packageNameFilter, this.includeSubpackages)) { + continue; + } + } classpathEntries.add(f); } else { logger.debug("path element does not exist {}",f); @@ -131,7 +139,7 @@ public class IterableClasspath extends CloseableFilterableJavaFileObjectIterable nextEntry = new ZipEntryJavaFileObject(openFile, openArchive, entry); } return; - } else if (nestedZip == null && entryName.startsWith(BOOT_PACKAGING_PREFIX_FOR_LIBRARIES) && entryName.endsWith(".jar")) { + } else if (nestedZip == null && entryName.startsWith(MemoryBasedJavaFileManager.BOOT_PACKAGING_PREFIX_FOR_LIBRARIES) && entryName.endsWith(".jar")) { // nested jar in uber jar logger.debug("opening nested archive {}",entry.getName()); ZipInputStream zis = new ZipInputStream(openArchive.getInputStream(entry)); @@ -211,4 +219,8 @@ public class IterableClasspath extends CloseableFilterableJavaFileObjectIterable } } + + public void reset() { + close(); + } } \ No newline at end of file diff --git a/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/MemoryBasedJavaFileManager.java b/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/MemoryBasedJavaFileManager.java index 0e5dabcce..ba1ac46f3 100644 --- a/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/MemoryBasedJavaFileManager.java +++ b/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/MemoryBasedJavaFileManager.java @@ -18,14 +18,24 @@ package org.springframework.cloud.function.compiler.java; import java.io.File; import java.io.IOException; +import java.net.URI; import java.net.URL; import java.net.URLClassLoader; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.StringTokenizer; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipInputStream; import javax.tools.FileObject; import javax.tools.JavaFileManager; @@ -37,6 +47,7 @@ import org.eclipse.aether.artifact.DefaultArtifact; import org.eclipse.aether.graph.Dependency; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.cloud.function.compiler.java.IterableClasspath.ZipEnumerator; /** * A file manager that serves source code from in memory and ensures output results are @@ -48,17 +59,30 @@ import org.slf4j.LoggerFactory; */ public class MemoryBasedJavaFileManager implements JavaFileManager { + final static String BOOT_PACKAGING_PREFIX_FOR_CLASSES = "BOOT-INF/classes/"; + + final static String BOOT_PACKAGING_PREFIX_FOR_LIBRARIES = "BOOT-INF/lib/"; + private static Logger logger = LoggerFactory .getLogger(MemoryBasedJavaFileManager.class); private CompilationOutputCollector outputCollector; - private List toClose = new ArrayList<>(); +// private List toClose = new ArrayList<>(); private Map resolvedAdditionalDependencies = new LinkedHashMap<>(); + + private String platformClasspath; + private String classpath; + + private CompilationInfoCache compilationInfoCache; + + private Map iterables = new HashMap<>(); + public MemoryBasedJavaFileManager() { outputCollector = new CompilationOutputCollector(); + compilationInfoCache = new CompilationInfoCache(); } @Override @@ -74,7 +98,184 @@ public class MemoryBasedJavaFileManager implements JavaFileManager { logger.debug("getClassLoader({})", location); return null; // Do not currently need to load plugins } + + // Holds information that may help speed up compilation + static class CompilationInfoCache { + private Map archivePackageCache; + + static class ArchiveInfo { + + // The packages identified in a particular archive + private List packageNames; + + private boolean isBootJar = false; + + public ArchiveInfo(List packageNames, boolean isBootJar) { + this.packageNames = packageNames; + Collections.sort(this.packageNames); + this.isBootJar = isBootJar; + } + + public List getPackageNames() { + return packageNames; + } + + public boolean isBootJar() { + return isBootJar; + } + + public boolean containsPackage(String packageName, boolean subpackageMatchesAllowed) { + if (subpackageMatchesAllowed) { + for (String candidatePackageName: packageNames) { + if (candidatePackageName.startsWith(packageName)) { + return true; + } + } + return false; + } else { + // Must be an exact match, fast binary search: + int pos = Collections.binarySearch(packageNames, packageName); + return (pos >= 0); + } + } + } + + ArchiveInfo getArchiveInfoFor(File archive) { + if (!archive.isFile() || !(archive.getName().endsWith(".zip") || archive.getName().endsWith(".jar"))) { + // it is not an archive + return null; + } + if (archivePackageCache == null) { + archivePackageCache = new HashMap<>(); + } + try { + ArchiveInfo result = archivePackageCache.get(archive); + if (result == null) { + result = buildArchiveInfo(archive); + archivePackageCache.put(archive, result); + } + return result; + } catch (Exception e) { + throw new IllegalStateException("Unexpected problem caching entries from "+archive.getName(), e); + } + } + + /** + * Walk the specified archive and collect up the package names of any .class files encountered. If + * the archive contains nested jars packaged in a BOOT style way (under a BOOT-INF/lib folder) then + * walk those too and include relevant packages. + * + * @param file archive file to discover packages from + * @return an ArchiveInfo encapsulating package info from the archive + */ + private ArchiveInfo buildArchiveInfo(File file) { + List packageNames = new ArrayList<>(); + boolean isBootJar = false; + try (ZipFile openArchive = new ZipFile(file)) { + Enumeration entries = openArchive.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + String name = entry.getName(); + if (name.endsWith(".class")) { + if (name.startsWith(BOOT_PACKAGING_PREFIX_FOR_CLASSES)) { + isBootJar = true; + int idx = name.lastIndexOf('/') + 1; + if (idx != 0 ) { + if (idx == BOOT_PACKAGING_PREFIX_FOR_CLASSES.length()) { + // default package + packageNames.add("/"); + } else { + // Normalize to forward slashes + name = name.substring(BOOT_PACKAGING_PREFIX_FOR_CLASSES.length(), idx); + name = name.replace('\\', '/'); + packageNames.add(name); + } + } + } else { + int idx = name.lastIndexOf('/') + 1; +// if (name.contains("TestX")) { +// System.out.println("For TestX: "+name+" "+idx+" "+BOOT_PACKAGING_PREFIX_FOR_CLASSES.length()); +// } + if (idx != 0 ) { + // Normalize to forward slashes + name = name.replace('\\', '/'); + name = name.substring(0, idx); + packageNames.add(name); + } else if (idx == 0) { + // default package entries in here + packageNames.add("/"); + } + } + } else if (name.startsWith(BOOT_PACKAGING_PREFIX_FOR_LIBRARIES) && name.endsWith(".jar")) { + isBootJar = true; + try (ZipInputStream zis = new ZipInputStream(openArchive.getInputStream(entry))) { + Enumeration nestedZipEnumerator = new ZipEnumerator(zis); + while (nestedZipEnumerator.hasMoreElements()) { + ZipEntry innerEntry = nestedZipEnumerator.nextElement(); + String innerEntryName = innerEntry.getName(); + if (innerEntryName.endsWith(".class")) { + int idx = innerEntryName.lastIndexOf('/') + 1; + if (idx != 0 ) { + // Normalize to forward slashes + innerEntryName = innerEntryName.replace('\\', '/'); + innerEntryName = innerEntryName.substring(0, idx); + packageNames.add(innerEntryName); + } else if (idx == 0) { + // default package entries in here + packageNames.add("/"); + } + } + } + } + } + } + } catch (IOException ioe) { + throw new IllegalStateException("Unexpected problem determining packages in "+file,ioe); + } + return new ArchiveInfo(packageNames, isBootJar); + } + + } + + static class Key { + private String classpath; + private String packageName; + private Set kinds; + private boolean recurse; + + public Key(String classpath, String packageName, Set kinds, boolean recurse) { + this.classpath = classpath; + this.packageName = packageName; + this.kinds = kinds; + this.recurse = recurse; + } + + @Override + public int hashCode() { + return ((classpath.hashCode()*37+(packageName==null?0:packageName.hashCode()))*37+kinds.hashCode())*37+(recurse?1:0); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Key)) { + return false; + } + Key that = (Key)obj; + return classpath.equals(that.classpath) && + kinds.equals(that.kinds) && + (recurse==that.recurse) && + (packageName==null?(that.packageName==null):this.packageName.equals(that.packageName)); + } + } + + private String getPlatformClassPath() { + if (platformClasspath == null) { + platformClasspath = System.getProperty("sun.boot.class.path"); + } + return platformClasspath; + } + @Override public Iterable list(Location location, String packageName, Set kinds, boolean recurse) throws IOException { @@ -82,7 +283,7 @@ public class MemoryBasedJavaFileManager implements JavaFileManager { String classpath = ""; if (location == StandardLocation.PLATFORM_CLASS_PATH && (kinds == null || kinds.contains(Kind.CLASS))) { - classpath = System.getProperty("sun.boot.class.path"); + classpath = getPlatformClassPath(); logger.debug("Creating iterable for boot class path: {}", classpath); } else if (location == StandardLocation.CLASS_PATH @@ -98,34 +299,69 @@ public class MemoryBasedJavaFileManager implements JavaFileManager { classpath = javaClassPath; logger.debug("Creating iterable for class path: {}", classpath); } - CloseableFilterableJavaFileObjectIterable resultIterable = new IterableClasspath( - classpath, packageName, recurse); - toClose.add(resultIterable); + Key k = new Key(classpath, packageName, kinds, recurse); + IterableClasspath resultIterable = iterables.get(k); + if (resultIterable == null) { + resultIterable = new IterableClasspath(compilationInfoCache, classpath, packageName, recurse); + iterables.put(k, resultIterable); + } + resultIterable.reset(); return resultIterable; } private String getClassPath() { - ClassLoader loader = InMemoryJavaFileObject.class.getClassLoader(); - if (loader instanceof URLClassLoader) { - URL[] urls = ((URLClassLoader) loader).getURLs(); - if (urls.length > 1) { // heuristic that catches Maven surefire tests - if (!urls[0].toString().startsWith("jar:file:")) { // heuristic for Spring Boot fat jar - StringBuilder builder = new StringBuilder(); - for (URL url : urls) { - if (builder.length() > 0) { - builder.append(File.pathSeparator); + if (classpath == null) { + ClassLoader loader = InMemoryJavaFileObject.class.getClassLoader(); + String cp = null; + if (loader instanceof URLClassLoader) { + URL[] urls = ((URLClassLoader) loader).getURLs(); + if (urls.length > 1) { // heuristic that catches Maven surefire tests + if (!urls[0].toString().startsWith("jar:file:")) { // heuristic for Spring Boot fat jar + StringBuilder builder = new StringBuilder(); + for (URL url : urls) { + if (builder.length() > 0) { + builder.append(File.pathSeparator); + } + String path = url.toString(); + if (path.startsWith("file:")) { + path = path.substring("file:".length()); + } + builder.append(path); } - String path = url.toString(); - if (path.startsWith("file:")) { - path = path.substring("file:".length()); - } - builder.append(path); + cp = builder.toString(); } - return builder.toString(); } } + if (cp == null) { + cp = System.getProperty("java.class.path"); + } + classpath = pathWithPlatformClassPathRemoved(cp); } - return System.getProperty("java.class.path"); + return classpath; + } + + // remove the platform classpath entries, they will be search separately (and earlier) + private String pathWithPlatformClassPathRemoved(String classpath) { + Set pcps = toList(getPlatformClassPath()); + Set cps = toList(classpath); + cps.removeAll(pcps); + StringBuilder builder = new StringBuilder(); + for (String cpe: cps) { + if (builder.length() > 0) { + builder.append(File.pathSeparator); + } + builder.append(cpe); + } + return builder.toString(); + } + + private Set toList(String path) { + Set result = new LinkedHashSet<>(); + StringTokenizer tokenizer = new StringTokenizer(path,File.pathSeparator); + while (tokenizer.hasMoreTokens()) { + result.add(tokenizer.nextToken()); + } + return result; } @Override @@ -201,8 +437,9 @@ public class MemoryBasedJavaFileManager implements JavaFileManager { @Override public void close() throws IOException { - for (CloseableFilterableJavaFileObjectIterable closeable : toClose) { - closeable.close(); + Collection toClose = iterables.values(); + for (IterableClasspath icp: toClose) { + icp.close(); } } @@ -232,6 +469,8 @@ public class MemoryBasedJavaFileManager implements JavaFileManager { CompilationMessage.Kind.ERROR, re.getMessage(), null, 0, 0); resolutionMessages.add(compilationMessage); } + } else if (dependency.startsWith("file:")) { + resolvedAdditionalDependencies.put(dependency, new File(URI.create(dependency))); } else { resolutionMessages.add(new CompilationMessage( diff --git a/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/RuntimeJavaCompiler.java b/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/RuntimeJavaCompiler.java index b8cb44b39..5b8a5d58e 100644 --- a/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/RuntimeJavaCompiler.java +++ b/spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/RuntimeJavaCompiler.java @@ -49,7 +49,7 @@ public class RuntimeJavaCompiler { * classes. * @param className the name of the class (dotted form, e.g. com.foo.bar.Goo) * @param classSourceCode the full source code for the class - * @param dependencies optional maven coordinates for dependencies "maven://groupId:artifactId:version" + * @param dependencies optional coordinates for dependencies, maven 'maven://groupId:artifactId:version', or 'file:' URIs for local files * @return a CompilationResult that encapsulates what happened during compilation (classes/messages produced) */ public CompilationResult compile(String className, String classSourceCode, String... dependencies) { diff --git a/spring-cloud-function-compiler/src/test/java/org/springframework/cloud/function/compiler/CompilerDependencyResolutionTests.java b/spring-cloud-function-compiler/src/test/java/org/springframework/cloud/function/compiler/CompilerDependencyResolutionTests.java new file mode 100644 index 000000000..dfff6b161 --- /dev/null +++ b/spring-cloud-function-compiler/src/test/java/org/springframework/cloud/function/compiler/CompilerDependencyResolutionTests.java @@ -0,0 +1,441 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.function.compiler; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.jar.JarEntry; +import java.util.jar.JarInputStream; +import java.util.jar.JarOutputStream; +import java.util.zip.ZipEntry; + +import org.junit.Test; +import org.slf4j.LoggerFactory; +import org.springframework.cloud.function.compiler.java.CompilationResult; +import org.springframework.cloud.function.compiler.java.RuntimeJavaCompiler; +import org.springframework.cloud.function.core.FunctionFactoryUtils; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.LoggerContext; + +/** + * Tests that verify dependency resolution. Dependencies can be resolved against simple + * classpath entries or against classes under BOOT-INF/classes or in a nested jar under + * under BOOT-INF/lib. Finding classes in those locations enables compilation + * against a packaged boot jar. + * + * @author Andy Clement + */ +public class CompilerDependencyResolutionTests { + + @Test + public void compilingTestClass() throws Exception { + ClassDescriptor t1 = compile("Test1","package com.test;\npublic class Test1 { public static String doit() { return \"T1\";}}\n"); + String result = (String) t1.clazz.getDeclaredMethod("doit").invoke(null); + assertEquals("T1",result); + } + + @Test + public void packagingClassesIntoJar() { + ClassDescriptor t1 = getTestClass("1"); + ClassDescriptor t2 = getTestClass("2"); + File jar = JarBuilder.create().addEntries(t1, t2).getJar(); + assertJarContents(jar, t1, t2); + } + + /** + * Doesn't actually verify the caching helps but can be useful to run to see + * current numbers. + */ + @Test + public void speedtest() { + LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory(); + Logger rootLogger = loggerContext.getLogger("org.springframework.cloud.function.compiler"); + rootLogger.setLevel(Level.ERROR); + + // 10 uses of a single function compiler: + long stime = System.currentTimeMillis(); + FunctionCompiler fc = new FunctionCompiler(String.class.getName()); + for (int i=0;i<5;i++) { + stime = System.currentTimeMillis(); + CompiledFunctionFactory> result = + fc.compile("foos", "flux -> flux.map(v -> v.toUpperCase())", "Flux", "Flux"); + assertThat(FunctionFactoryUtils.isFluxFunction(result.getFactoryMethod())).isTrue(); + System.out.println("Reusing FunctionCompiler: #"+(i+1)+" = "+(System.currentTimeMillis()-stime)+"ms"); + } + + // 3 separate FunctionCompilers: + stime = System.currentTimeMillis(); + CompiledFunctionFactory> compiled = new FunctionCompiler( + String.class.getName()).compile("foos", "flux -> flux.map(v -> v.toUpperCase())", "Flux", + "Flux"); + assertThat(FunctionFactoryUtils.isFluxFunction(compiled.getFactoryMethod())).isTrue(); + long etime = System.currentTimeMillis(); + long time1 = (etime - stime); + System.out.println("New FunctionCompiler: "+time1+"ms"); + + stime = System.currentTimeMillis(); + compiled = new FunctionCompiler(String.class.getName()).compile("foos", + "flux -> flux.map(v -> v.toUpperCase())", "Flux", "Flux"); + assertThat(FunctionFactoryUtils.isFluxFunction(compiled.getFactoryMethod())).isTrue(); + etime = System.currentTimeMillis(); + long time2 = (etime - stime); + System.out.println("New FunctionCompiler: "+time2+"ms"); + + stime = System.currentTimeMillis(); + compiled = new FunctionCompiler(String.class.getName()).compile("foos", + "flux -> flux.map(v -> v.toUpperCase())", "Flux", "Flux"); + assertThat(FunctionFactoryUtils.isFluxFunction(compiled.getFactoryMethod())).isTrue(); + etime = System.currentTimeMillis(); + long time3 = (etime - stime); + System.out.println("New FunctionCompiler: "+time3+"ms"); + } + + @Test + public void usingJarNoPackageDecl() throws Exception { + ClassDescriptor tx = compile("TestX","public class TestX { public static String doit() { return \"TX\";}}\n"); + File jar = JarBuilder.create().addEntry(tx).getJar(); + assertJarContents(jar, tx); + CompilationResult result = new RuntimeJavaCompiler().compile("A", + "public class A {\n" + + " public static Object run() {\n" + + " return new TestX();\n" + + " }\n" + + "}", + jar.toURI().toString()); + assertTrue("Should be no problems: "+result.getCompilationMessages(), result.getCompilationMessages().isEmpty()); + try (URLClassLoader cl = new TestClassLoader(tx,descriptorFromResult(result))) { + Class class1 = cl.loadClass("A"); + Object invoke = class1.getDeclaredMethod("run").invoke(null); + assertEquals(tx.name, invoke.getClass().getName()); + } + } + + + // A class with no package declaration is placed under BOOT-INF/classes/ in a jar that is then used for resolution + @Test + public void usingJarNoPackageDeclBootInfClasses() throws Exception { + ClassDescriptor t1 = compile("TestX","public class TestX { public static String doit() { return \"TX\";}}\n"); + File jar = JarBuilder.create().addEntryWithPrefix("BOOT-INF/classes/",t1).getJar(); + assertJarContents(jar, "BOOT-INF/classes/", t1); + CompilationResult result = new RuntimeJavaCompiler().compile("A", + "public class A {\n" + + " public static Object run() {\n" + + " return new TestX();\n" + + " }\n" + + "}", + jar.toURI().toString()); + assertTrue("Should be no problems: "+result.getCompilationMessages(), result.getCompilationMessages().isEmpty()); + try (URLClassLoader cl = new TestClassLoader(t1,descriptorFromResult(result))) { + Class class1 = cl.loadClass("A"); + Object invoke = class1.getDeclaredMethod("run").invoke(null); + assertEquals(t1.name, invoke.getClass().getName()); + } + } + + // A class with no package declaration is placed in a jar which is then placed under + // under BOOT-INF/lib/ in a jar that is then used for resolution + @Test + public void usingJarNoPackageDeclNestedBootInfLib() throws Exception { + ClassDescriptor t1 = compile("TestX","public class TestX { public static String doit() { return \"TX\";}}\n"); + File jar = JarBuilder.create().addEntry(t1).getJar(); + assertJarContents(jar, t1); + // Now stick that jar in another jar! + File jar2 = JarBuilder.create().addEntry("BOOT-INF/lib/inner.jar",jar).getJar(); + CompilationResult result = new RuntimeJavaCompiler().compile("A", + "public class A {\n" + + " public static Object run() {\n" + + " return new TestX();\n" + + " }\n" + + "}", + jar2.toURI().toString()); + assertTrue("Should be no problems: "+result.getCompilationMessages(), result.getCompilationMessages().isEmpty()); + try (URLClassLoader cl = new TestClassLoader(t1,descriptorFromResult(result))) { + Class class1 = cl.loadClass("A"); + Object invoke = class1.getDeclaredMethod("run").invoke(null); + assertEquals(t1.name, invoke.getClass().getName()); + } + } + + + // Build a jar containing a type with a package declaration and building against it + @Test + public void usingJarWithPackageDecl() throws Exception { + ClassDescriptor t1 = getTestClass("1"); + File jar = JarBuilder.create().addEntry(t1).getJar(); + assertJarContents(jar, t1); + CompilationResult result = new RuntimeJavaCompiler().compile("A", + "import " + t1.name.replace('$', '.') + ";\n" + + "public class A {\n" + + " public static Object run() {\n" + + " return new Test1();\n" + + " }\n" + + "}", + jar.toURI().toString()); + assertTrue("Should be no problems: "+result.getCompilationMessages(), result.getCompilationMessages().isEmpty()); + try (URLClassLoader cl = new TestClassLoader(t1,descriptorFromResult(result))) { + Class class1 = cl.loadClass("A"); + Object invoke = class1.getDeclaredMethod("run").invoke(null); + assertEquals(t1.name, invoke.getClass().getName()); + } + } + + @Test + public void usingJarWithPackageDeclBootInfClasses() throws Exception { + // Here the dependencies are under BOOT-INF/classes in the jar + ClassDescriptor t1 = getTestClass("1"); + File jar = JarBuilder.create().addEntryWithPrefix("BOOT-INF/classes/",t1).getJar(); + assertJarContents(jar, "BOOT-INF/classes/", t1); + CompilationResult result = new RuntimeJavaCompiler().compile("A", + "import " + t1.name.replace('$', '.') + ";\n" + + "public class A {\n" + + " public static Object run() {\n" + + " return new Test1();\n" + + " }\n" + + "}", + jar.toURI().toString()); + assertTrue("Should be no problems: "+result.getCompilationMessages(), result.getCompilationMessages().isEmpty()); + try (URLClassLoader cl = new TestClassLoader(t1,descriptorFromResult(result))) { + Class class1 = cl.loadClass("A"); + Object invoke = class1.getDeclaredMethod("run").invoke(null); + assertEquals(t1.name, invoke.getClass().getName()); + } + } + + + @Test + public void usingJarWithPackageDeclNestedBootInfLib() throws Exception { + // Here the dependencies are under BOOT-INF/lib/nested.jar + ClassDescriptor t1 = getTestClass("1"); + File jar = JarBuilder.create().addEntry(t1).getJar(); + assertJarContents(jar, t1); + // Now stick that jar in another jar! + File jar2 = JarBuilder.create().addEntry("BOOT-INF/lib/inner.jar",jar).getJar(); + CompilationResult result = new RuntimeJavaCompiler().compile("A", + "import " + t1.name.replace('$', '.') + ";\n" + + "public class A {\n" + + " public static Object run() {\n" + + " return new Test1();\n" + + " }\n" + + "}", + jar2.toURI().toString()); + assertTrue("Should be no problems: "+result.getCompilationMessages(), result.getCompilationMessages().isEmpty()); + try (URLClassLoader cl = new TestClassLoader(t1,descriptorFromResult(result))) { + Class class1 = cl.loadClass("A"); + Object invoke = class1.getDeclaredMethod("run").invoke(null); + assertEquals(t1.name, invoke.getClass().getName()); + } + } + + // --- + + // Simple classloader that can load from descriptors + class TestClassLoader extends URLClassLoader { + + ClassDescriptor[] descriptors; + + public TestClassLoader(ClassDescriptor... descriptors) { + super(new URL[0], TestClassLoader.class.getClassLoader()); + this.descriptors = descriptors; + } + + @Override + protected Class findClass(String name) throws ClassNotFoundException { + for (ClassDescriptor descriptor: descriptors) { + if (descriptor.name.equals(name)) { + return defineClass(descriptor.name, descriptor.bytes, 0, descriptor.bytes.length); + } + } + return super.findClass(name); + } + + } + + // Simple holder for the result of compilation + static class ClassDescriptor { + final String name; + final byte[] bytes; + final Class clazz; + + public ClassDescriptor(String name, byte[] bytes, Class clazz) { + this.name = name; + this.bytes = bytes; + this.clazz = clazz; + } + } + + private ClassDescriptor descriptorFromResult(CompilationResult result) { + Class clazz = result.getCompiledClasses().get(0); + return new ClassDescriptor(clazz.getName(),result.getClassBytes(clazz.getName()),clazz); + } + + private ClassDescriptor compile(String className, String classSourceCode) { + CompilationResult compile = new RuntimeJavaCompiler().compile(className, classSourceCode); + assertTrue("Should be empty: \n"+compile.getCompilationMessages(), compile.getCompilationMessages().isEmpty()); + Class clazz = compile.getCompiledClasses().get(0); + return new ClassDescriptor(clazz.getName(),compile.getClassBytes(clazz.getName()), compile.getCompiledClasses().get(0)); + } + + private ClassDescriptor getTestClass(String suffix) { + try { + return compile("Test"+suffix,"package com.test;\npublic class Test"+suffix+" { public static String doit() { return \"T"+suffix+"\";}}\n"); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + private void assertJarContents(File jar, ClassDescriptor... classdescriptors) { + assertJarContents(jar, "", classdescriptors); + } + + private void assertJarContents(File jar, String prefix, ClassDescriptor... classDescriptors) { + List clazzes = new ArrayList<>(); + for (ClassDescriptor classDescriptor : classDescriptors) { + clazzes.add(prefix + classDescriptor.name.replace('.', '/') + ".class"); + } + walkJar(jar, (entry) -> clazzes.remove(entry.getName())); + assertTrue("Should be empty: " + clazzes, clazzes.isEmpty()); + } + + private void walkJar(File jar, Consumer fn) { + try { + JarInputStream jarInputStream = new JarInputStream(new FileInputStream(jar)); + while (true) { + JarEntry nextJarEntry = jarInputStream.getNextJarEntry(); + if (nextJarEntry == null) { + break; + } + fn.accept(nextJarEntry); + } + jarInputStream.close(); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + } + + @SuppressWarnings("unused") + private void printJar(File jar) { + System.out.println("Contents of jar: " + jar); + walkJar(jar, (entry) -> { + System.out.println("- " + entry.getName()); + }); + } + + static class JarBuilder { + + File jarFile; + JarOutputStream jos; + + private JarBuilder() { + try { + File newJar = File.createTempFile("test", ".jar"); + jarFile = newJar.getAbsoluteFile(); + newJar.delete(); + jos = new JarOutputStream(new FileOutputStream(jarFile)); + jarFile.deleteOnExit(); + } catch (IOException e) { + throw new IllegalStateException("Unexpected problem creating file", e); + } + } + + public JarBuilder addEntry(String entryName, File entryContentFile) { + try { + ZipEntry ze = new ZipEntry(entryName); + jos.putNextEntry(ze); + jos.write(loadBytes(entryContentFile)); + jos.closeEntry(); + return this; + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private byte[] loadBytes(File f) { + try (InputStream is = new FileInputStream(f)) { + byte[] bs = null; + byte[] buf = new byte[10000]; + int readCount; + while ((readCount = is.read(buf)) != -1) { + if (bs == null) { + bs = new byte[readCount]; + System.arraycopy(buf, 0, bs, 0, readCount); + } else { + byte[] newbs = new byte[bs.length + readCount]; + System.arraycopy(bs, 0, newbs, 0, bs.length); + System.arraycopy(buf, 0, newbs, bs.length, readCount); + bs = newbs; + } + } + return bs; + } catch (IOException ioe) { + throw new IllegalStateException(ioe); + } + } + + public JarBuilder addEntries(ClassDescriptor... classes) { + for (ClassDescriptor clazz : classes) { + addEntry(clazz); + } + return this; + } + + public JarBuilder addEntry(ClassDescriptor clazz) { + return addEntryWithPrefix("", clazz); + } + + public JarBuilder addEntryWithPrefix(String prefix, ClassDescriptor holder) { + try { + String n = holder.name.replace('.', '/') + ".class"; + ZipEntry ze = new ZipEntry(prefix + n); + jos.putNextEntry(ze); + jos.write(holder.bytes); + jos.closeEntry(); + return this; + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + public static JarBuilder create() { + return new JarBuilder(); + } + + private File getJar() { + try { + jos.close(); + } catch (IOException e) { + throw new IllegalStateException("Unable to close jar", e); + } + return jarFile; + } + + } + +}