Faster compilation of functions

This change includes caching and smarter analysis of classpaths
to limit the amount of jar walking necessary to find dependencies
when compiling. It also includes some new tests that verify
packaging of dependencies in boot style form (BOOT-INF/lib and
BOOT-INF/classes).

spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/CloseableFilterableJavaFileObjectIterable.java

@@ -20,6 +20,8 @@ import java.io.File;
 
 import javax.tools.JavaFileObject;
 
+import org.springframework.cloud.function.compiler.java.MemoryBasedJavaFileManager.CompilationInfoCache;
+
 /**
  * Common superclass for iterables that need to handle closing when finished
  * with and that need to handle possible constraints on the values that
@@ -35,22 +37,26 @@ public abstract class CloseableFilterableJavaFileObjectIterable implements Itera
 	private final static String BOOT_PACKAGING_PREFIX_FOR_CLASSES = "BOOT-INF/classes/";
 
 	// If set specifies the package the iterator consumer is interested in. Only
-	// return results in this package.
-	private String packageNameFilter;
+	// return results in this package. Will have a trailing separator to speed
+	// matching. '/' on its own represents the default package
+	protected String packageNameFilter;
 
 	// Indicates whether the consumer of the iterator wants to see classes
 	// that are in subpackages of those matching the filter.
-	private boolean includeSubpackages;
+	protected boolean includeSubpackages;
+	
+	protected CompilationInfoCache compilationInfoCache;
 
-	public CloseableFilterableJavaFileObjectIterable(String packageNameFilter, boolean includeSubpackages) {
+	public CloseableFilterableJavaFileObjectIterable(CompilationInfoCache compilationInfoCache, String packageNameFilter, boolean includeSubpackages) {
 		if (packageNameFilter!=null && packageNameFilter.contains(File.separator)) {
 			throw new IllegalArgumentException("Package name filters should use dots to separate components: "+packageNameFilter);
 		}
+		this.compilationInfoCache = compilationInfoCache;
 		// Normalize filter to forward slashes
 		this.packageNameFilter = packageNameFilter==null?null:packageNameFilter.replace('.', '/') + '/';
 		this.includeSubpackages = includeSubpackages;
 	}
-	
+
 	/**
 	 * Used by subclasses to check values against any specified constraints.
 	 * 
@@ -68,6 +74,16 @@ public abstract class CloseableFilterableJavaFileObjectIterable implements Itera
 		boolean accept;
 		// Normalize to forward slashes (some jars are producing paths with forward slashes, some with backward slashes)
 		name = name.replace('\\', '/');
+		if (packageNameFilter.length() == 1 && packageNameFilter.equals("/")) {
+			// This is the 'default package' filter representation
+			if (name.indexOf('/') == -1) {
+				accept = true;
+			} else if (BOOT_PACKAGING_AWARE) {
+					accept = name.startsWith(BOOT_PACKAGING_PREFIX_FOR_CLASSES) &&
+							name.indexOf('/',BOOT_PACKAGING_PREFIX_FOR_CLASSES.length()) == -1;
+			}
+			return accept;
+		}
 		if (includeSubpackages == true) {
 			accept = name.startsWith(packageNameFilter);
 			if (!accept && BOOT_PACKAGING_AWARE) {

spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/IterableClasspath.java

@@ -33,6 +33,8 @@ import javax.tools.JavaFileObject;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.cloud.function.compiler.java.MemoryBasedJavaFileManager.CompilationInfoCache;
+import org.springframework.cloud.function.compiler.java.MemoryBasedJavaFileManager.CompilationInfoCache.ArchiveInfo;
 
 /**
  * Iterable that will produce an iterator that returns classes found
@@ -46,24 +48,30 @@ public class IterableClasspath extends CloseableFilterableJavaFileObjectIterable
 
 	private static Logger logger = LoggerFactory.getLogger(IterableClasspath.class);
 	
-	private static final String BOOT_PACKAGING_PREFIX_FOR_LIBRARIES = "BOOT-INF/lib/";
-	
 	private List<File> classpathEntries = new ArrayList<>();
 	
 	private List<ZipFile> openArchives = new ArrayList<>();
 
 	/**
+	 * @param compilationInfoCache cache of info that may help accelerate compilation
 	 * @param classpath a classpath of jars/directories
 	 * @param packageNameFilter an optional package name if choosing to filter (e.g. com.example)
 	 * @param includeSubpackages if true, include results in subpackages of the specified package filter
 	 */
-	IterableClasspath(String classpath, String packageNameFilter, boolean includeSubpackages) {
-		super(packageNameFilter, includeSubpackages);
+	IterableClasspath(CompilationInfoCache compilationInfoCache, String classpath, String packageNameFilter, boolean includeSubpackages) {
+		super(compilationInfoCache, packageNameFilter, includeSubpackages);
 		StringTokenizer tokenizer = new StringTokenizer(classpath, File.pathSeparator);
 		while (tokenizer.hasMoreElements()) {
 			String nextEntry = tokenizer.nextToken();
 			File f = new File(nextEntry);
 			if (f.exists()) {
+				// Skip iterating over archives that cannot possibly match the filter
+				if (this.packageNameFilter != null && this.packageNameFilter.length() > 0) {
+					ArchiveInfo archiveInfo = compilationInfoCache.getArchiveInfoFor(f);
+					if (archiveInfo != null && !archiveInfo.containsPackage(this.packageNameFilter, this.includeSubpackages)) {
+						continue;
+					}
+				}
 				classpathEntries.add(f);
 			} else {
 				logger.debug("path element does not exist {}",f);
@@ -131,7 +139,7 @@ public class IterableClasspath extends CloseableFilterableJavaFileObjectIterable
 											nextEntry = new ZipEntryJavaFileObject(openFile, openArchive, entry);
 										}
 										return;
-									} else if (nestedZip == null && entryName.startsWith(BOOT_PACKAGING_PREFIX_FOR_LIBRARIES) && entryName.endsWith(".jar")) {
+									} else if (nestedZip == null && entryName.startsWith(MemoryBasedJavaFileManager.BOOT_PACKAGING_PREFIX_FOR_LIBRARIES) && entryName.endsWith(".jar")) {
 										// nested jar in uber jar
 										logger.debug("opening nested archive {}",entry.getName());
 										ZipInputStream zis = new ZipInputStream(openArchive.getInputStream(entry));
@@ -211,4 +219,8 @@ public class IterableClasspath extends CloseableFilterableJavaFileObjectIterable
 		}
 		
 	}
+
+	public void reset() {
+		close();
+	}
 }

spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/MemoryBasedJavaFileManager.java

@@ -18,14 +18,24 @@ package org.springframework.cloud.function.compiler.java;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+import java.util.zip.ZipInputStream;
 
 import javax.tools.FileObject;
 import javax.tools.JavaFileManager;
@@ -37,6 +47,7 @@ import org.eclipse.aether.artifact.DefaultArtifact;
 import org.eclipse.aether.graph.Dependency;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.cloud.function.compiler.java.IterableClasspath.ZipEnumerator;
 
 /**
  * A file manager that serves source code from in memory and ensures output results are
@@ -48,17 +59,30 @@ import org.slf4j.LoggerFactory;
  */
 public class MemoryBasedJavaFileManager implements JavaFileManager {
 
+	final static String BOOT_PACKAGING_PREFIX_FOR_CLASSES = "BOOT-INF/classes/";
+
+	final static String BOOT_PACKAGING_PREFIX_FOR_LIBRARIES = "BOOT-INF/lib/";
+	
 	private static Logger logger = LoggerFactory
 			.getLogger(MemoryBasedJavaFileManager.class);
 
 	private CompilationOutputCollector outputCollector;
 
-	private List<CloseableFilterableJavaFileObjectIterable> toClose = new ArrayList<>();
+//	private List<CloseableFilterableJavaFileObjectIterable> toClose = new ArrayList<>();
 
 	private Map<String, File> resolvedAdditionalDependencies = new LinkedHashMap<>();
+	
+	private String platformClasspath;
 
+	private String classpath;
+
+	private CompilationInfoCache compilationInfoCache;
+	
+	private Map<Key, IterableClasspath> iterables = new HashMap<>();
+	
 	public MemoryBasedJavaFileManager() {
 		outputCollector = new CompilationOutputCollector();
+		compilationInfoCache = new CompilationInfoCache();
 	}
 
 	@Override
@@ -74,7 +98,184 @@ public class MemoryBasedJavaFileManager implements JavaFileManager {
 		logger.debug("getClassLoader({})", location);
 		return null; // Do not currently need to load plugins
 	}
+	
+	// Holds information that may help speed up compilation
+	static class CompilationInfoCache {
 
+		private Map<File, ArchiveInfo> archivePackageCache;
+		
+		static class ArchiveInfo {
+			
+			// The packages identified in a particular archive
+			private List<String> packageNames;
+			
+			private boolean isBootJar = false;
+			
+			public ArchiveInfo(List<String> packageNames, boolean isBootJar) {
+				this.packageNames = packageNames;
+				Collections.sort(this.packageNames);
+				this.isBootJar = isBootJar;
+			}
+
+			public List<String> getPackageNames() {
+				return packageNames;
+			}
+
+			public boolean isBootJar() {
+				return isBootJar;
+			}
+
+			public boolean containsPackage(String packageName, boolean subpackageMatchesAllowed) {
+				if (subpackageMatchesAllowed) {
+					for (String candidatePackageName: packageNames) {
+						if (candidatePackageName.startsWith(packageName)) {
+							return true;
+						}
+					}
+					return false;
+				} else {
+					// Must be an exact match, fast binary search:
+					int pos = Collections.binarySearch(packageNames, packageName);
+					return  (pos >= 0);
+				}
+			}
+		}
+		
+		ArchiveInfo getArchiveInfoFor(File archive) {
+			if (!archive.isFile() || !(archive.getName().endsWith(".zip") || archive.getName().endsWith(".jar"))) {
+				// it is not an archive
+				return null;
+			}
+			if (archivePackageCache == null) {
+				archivePackageCache = new HashMap<>();
+			}
+			try {
+				ArchiveInfo result = archivePackageCache.get(archive);
+				if (result == null) {
+					result = buildArchiveInfo(archive);
+					archivePackageCache.put(archive, result);
+				}
+				return result;
+			} catch (Exception e) {
+				throw new IllegalStateException("Unexpected problem caching entries from "+archive.getName(), e);
+			}
+		}
+		
+		/**
+		 * Walk the specified archive and collect up the package names of any .class files encountered. If
+		 * the archive contains nested jars packaged in a BOOT style way (under a BOOT-INF/lib folder) then
+		 * walk those too and include relevant packages.
+		 * 
+		 * @param file archive file to discover packages from
+		 * @return an ArchiveInfo encapsulating package info from the archive
+		 */
+		private ArchiveInfo buildArchiveInfo(File file) {
+			List<String> packageNames = new ArrayList<>();
+			boolean isBootJar = false;
+			try (ZipFile openArchive = new ZipFile(file)) {
+				Enumeration<? extends ZipEntry> entries = openArchive.entries();
+				while (entries.hasMoreElements()) {
+					ZipEntry entry = entries.nextElement();
+					String name = entry.getName();
+					if (name.endsWith(".class")) {
+						if (name.startsWith(BOOT_PACKAGING_PREFIX_FOR_CLASSES)) {
+							isBootJar = true;
+							int idx = name.lastIndexOf('/') + 1;
+							if (idx != 0 ) {
+								if (idx == BOOT_PACKAGING_PREFIX_FOR_CLASSES.length()) {
+									// default package
+									packageNames.add("/");
+								} else {
+									// Normalize to forward slashes
+									name = name.substring(BOOT_PACKAGING_PREFIX_FOR_CLASSES.length(), idx);
+									name = name.replace('\\', '/');
+									packageNames.add(name);
+								}
+							}
+						} else {
+							int idx = name.lastIndexOf('/') + 1;
+//							if (name.contains("TestX")) {
+//								System.out.println("For TestX: "+name+" "+idx+" "+BOOT_PACKAGING_PREFIX_FOR_CLASSES.length());
+//							}
+							if (idx != 0 ) {
+								// Normalize to forward slashes
+								name = name.replace('\\', '/');
+								name = name.substring(0, idx);
+								packageNames.add(name);
+							} else if (idx == 0) {
+								// default package entries in here
+								packageNames.add("/");
+							}
+						}
+					} else if (name.startsWith(BOOT_PACKAGING_PREFIX_FOR_LIBRARIES) && name.endsWith(".jar")) {
+						isBootJar = true;
+						try (ZipInputStream zis = new ZipInputStream(openArchive.getInputStream(entry))) {
+							Enumeration<? extends ZipEntry> nestedZipEnumerator = new ZipEnumerator(zis);
+							while (nestedZipEnumerator.hasMoreElements()) {
+								ZipEntry innerEntry = nestedZipEnumerator.nextElement();
+								String innerEntryName = innerEntry.getName();
+								if (innerEntryName.endsWith(".class")) {
+									int idx = innerEntryName.lastIndexOf('/') + 1;
+									if (idx != 0 ) {
+										// Normalize to forward slashes
+										innerEntryName = innerEntryName.replace('\\', '/');
+										innerEntryName = innerEntryName.substring(0, idx);
+										packageNames.add(innerEntryName);
+									} else if (idx == 0) {
+										// default package entries in here
+										packageNames.add("/");
+									}
+								}
+							}
+						}
+					}
+				}
+			} catch (IOException ioe) {
+				throw new IllegalStateException("Unexpected problem determining packages in "+file,ioe);
+			}
+			return new ArchiveInfo(packageNames, isBootJar);
+		}
+
+	}
+
+	static class Key {
+		private String classpath;
+		private String packageName;
+		private Set<Kind> kinds;
+		private boolean recurse;
+		
+		public Key(String classpath, String packageName, Set<Kind> kinds, boolean recurse) {
+			this.classpath = classpath;
+			this.packageName = packageName;
+			this.kinds = kinds;
+			this.recurse = recurse;
+		}
+		
+		@Override
+		public int hashCode() {
+			return ((classpath.hashCode()*37+(packageName==null?0:packageName.hashCode()))*37+kinds.hashCode())*37+(recurse?1:0);
+		}
+	
+		@Override
+		public boolean equals(Object obj) {
+			if (!(obj instanceof Key)) {
+				return false;
+			}
+			Key that = (Key)obj;
+			return classpath.equals(that.classpath) &&
+					kinds.equals(that.kinds) &&
+					(recurse==that.recurse) &&
+					(packageName==null?(that.packageName==null):this.packageName.equals(that.packageName));
+		}
+	}
+
+	private String getPlatformClassPath() {
+		if (platformClasspath == null) {			
+			platformClasspath = System.getProperty("sun.boot.class.path");
+		}
+		return platformClasspath;
+	}
+	
 	@Override
 	public Iterable<JavaFileObject> list(Location location, String packageName,
 			Set<Kind> kinds, boolean recurse) throws IOException {
@@ -82,7 +283,7 @@ public class MemoryBasedJavaFileManager implements JavaFileManager {
 		String classpath = "";
 		if (location == StandardLocation.PLATFORM_CLASS_PATH
 				&& (kinds == null || kinds.contains(Kind.CLASS))) {
-			classpath = System.getProperty("sun.boot.class.path");
+			classpath = getPlatformClassPath();
 			logger.debug("Creating iterable for boot class path: {}", classpath);
 		}
 		else if (location == StandardLocation.CLASS_PATH
@@ -98,34 +299,69 @@ public class MemoryBasedJavaFileManager implements JavaFileManager {
 			classpath = javaClassPath;
 			logger.debug("Creating iterable for class path: {}", classpath);
 		}
-		CloseableFilterableJavaFileObjectIterable resultIterable = new IterableClasspath(
-				classpath, packageName, recurse);
-		toClose.add(resultIterable);
+		Key k = new Key(classpath, packageName, kinds, recurse);
+		IterableClasspath resultIterable = iterables.get(k);
+		if (resultIterable == null) {
+			resultIterable = new IterableClasspath(compilationInfoCache, classpath, packageName, recurse);
+			iterables.put(k, resultIterable);
+		}
+		resultIterable.reset();
 		return resultIterable;
 	}
 
 	private String getClassPath() {
-		ClassLoader loader = InMemoryJavaFileObject.class.getClassLoader();
-		if (loader instanceof URLClassLoader) {
-			URL[] urls = ((URLClassLoader) loader).getURLs();
-			if (urls.length > 1) { // heuristic that catches Maven surefire tests
-				if (!urls[0].toString().startsWith("jar:file:")) { // heuristic for Spring Boot fat jar
-					StringBuilder builder = new StringBuilder();
-					for (URL url : urls) {
-						if (builder.length() > 0) {
-							builder.append(File.pathSeparator);
+		if (classpath == null) {
+			ClassLoader loader = InMemoryJavaFileObject.class.getClassLoader();
+			String cp = null;
+			if (loader instanceof URLClassLoader) {
+				URL[] urls = ((URLClassLoader) loader).getURLs();
+				if (urls.length > 1) { // heuristic that catches Maven surefire tests
+					if (!urls[0].toString().startsWith("jar:file:")) { // heuristic for Spring Boot fat jar
+						StringBuilder builder = new StringBuilder();
+						for (URL url : urls) {
+							if (builder.length() > 0) {
+								builder.append(File.pathSeparator);
+							}
+							String path = url.toString();
+							if (path.startsWith("file:")) {
+								path = path.substring("file:".length());
+							}
+							builder.append(path);
 						}
-						String path = url.toString();
-						if (path.startsWith("file:")) {
-							path = path.substring("file:".length());
-						}
-						builder.append(path);
+						cp = builder.toString();
 					}
-					return builder.toString();
 				}
 			}
+			if (cp == null) {
+				cp = System.getProperty("java.class.path");
+			}
+			classpath = pathWithPlatformClassPathRemoved(cp);
 		}
-		return System.getProperty("java.class.path");
+		return classpath;
+	}
+
+	// remove the platform classpath entries, they will be search separately (and earlier)
+	private String pathWithPlatformClassPathRemoved(String classpath) {
+		Set<String> pcps = toList(getPlatformClassPath());
+		Set<String> cps = toList(classpath);
+		cps.removeAll(pcps);
+		StringBuilder builder = new StringBuilder();
+		for (String cpe: cps) {
+			if (builder.length() > 0) {
+				builder.append(File.pathSeparator);
+			}
+			builder.append(cpe);
+		}
+		return builder.toString();
+	}
+
+	private Set<String> toList(String path) {
+		Set<String> result = new LinkedHashSet<>();
+		StringTokenizer tokenizer = new StringTokenizer(path,File.pathSeparator);
+		while (tokenizer.hasMoreTokens()) {
+			result.add(tokenizer.nextToken());
+		}
+		return result;
 	}
 
 	@Override
@@ -201,8 +437,9 @@ public class MemoryBasedJavaFileManager implements JavaFileManager {
 
 	@Override
 	public void close() throws IOException {
-		for (CloseableFilterableJavaFileObjectIterable closeable : toClose) {
-			closeable.close();
+		Collection<IterableClasspath> toClose = iterables.values();
+		for (IterableClasspath icp: toClose) {
+			icp.close();
 		}
 	}
 
@@ -232,6 +469,8 @@ public class MemoryBasedJavaFileManager implements JavaFileManager {
 							CompilationMessage.Kind.ERROR, re.getMessage(), null, 0, 0);
 					resolutionMessages.add(compilationMessage);
 				}
+			} else if (dependency.startsWith("file:")) {
+				resolvedAdditionalDependencies.put(dependency, new File(URI.create(dependency)));
 			}
 			else {
 				resolutionMessages.add(new CompilationMessage(

spring-cloud-function-compiler/src/main/java/org/springframework/cloud/function/compiler/java/RuntimeJavaCompiler.java

@@ -49,7 +49,7 @@ public class RuntimeJavaCompiler {
 	 * classes.
 	 * @param className the name of the class (dotted form, e.g. com.foo.bar.Goo)
 	 * @param classSourceCode the full source code for the class
-	 * @param dependencies optional maven coordinates for dependencies "maven://groupId:artifactId:version"
+	 * @param dependencies optional coordinates for dependencies, maven 'maven://groupId:artifactId:version', or 'file:' URIs for local files
 	 * @return a CompilationResult that encapsulates what happened during compilation (classes/messages produced)
 	 */
 	public CompilationResult compile(String className, String classSourceCode, String... dependencies) {