switched scanning of affetced files to bulk scan mechanism to optimize performance

This commit is contained in:
Martin Lippert
2020-02-13 10:06:39 +01:00
parent aa0b33f70a
commit 2be05a4ac9
5 changed files with 127 additions and 198 deletions

View File

@@ -11,7 +11,6 @@
package org.springframework.ide.vscode.boot.app;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
@@ -30,11 +29,9 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.io.FileUtils;
import org.eclipse.lsp4j.SymbolInformation;
import org.eclipse.lsp4j.TextDocumentIdentifier;
import org.slf4j.Logger;
@@ -46,6 +43,7 @@ import org.springframework.ide.vscode.boot.java.annotations.AnnotationHierarchyA
import org.springframework.ide.vscode.boot.java.handlers.EnhancedSymbolInformation;
import org.springframework.ide.vscode.boot.java.handlers.SymbolAddOnInformation;
import org.springframework.ide.vscode.boot.java.handlers.SymbolProvider;
import org.springframework.ide.vscode.boot.java.utils.DocumentDescriptor;
import org.springframework.ide.vscode.boot.java.utils.SpringIndexer;
import org.springframework.ide.vscode.boot.java.utils.SpringIndexerJava;
import org.springframework.ide.vscode.boot.java.utils.SpringIndexerXML;
@@ -54,7 +52,6 @@ import org.springframework.ide.vscode.boot.java.utils.SpringIndexerXMLNamespaceH
import org.springframework.ide.vscode.boot.java.utils.SymbolCache;
import org.springframework.ide.vscode.boot.java.utils.SymbolHandler;
import org.springframework.ide.vscode.boot.java.utils.SymbolIndexConfig;
import org.springframework.ide.vscode.boot.java.utils.UpdatedDoc;
import org.springframework.ide.vscode.commons.java.IJavaProject;
import org.springframework.ide.vscode.commons.java.SpringProjectUtil;
import org.springframework.ide.vscode.commons.languageserver.java.JavaProjectFinder;
@@ -331,8 +328,8 @@ public class SpringSymbolIndex implements InitializingBean {
if (maybeProject.isPresent()) {
try {
UpdatedDoc newDoc = createUpdatedDoc(docURI, null);
futures.add(updateItems(maybeProject.get(), new UpdatedDoc[] {newDoc}, indexer));
DocumentDescriptor newDoc = createUpdatedDoc(docURI);
futures.add(updateItems(maybeProject.get(), new DocumentDescriptor[] {newDoc}, indexer));
}
catch (Exception e) {
log.error("", e);
@@ -359,7 +356,7 @@ public class SpringSymbolIndex implements InitializingBean {
List<String> docs = projectMapping.get(project);
try {
UpdatedDoc[] updatedDocs = docs.stream().map(doc -> createUpdatedDoc(doc, null)).toArray(UpdatedDoc[]::new);
DocumentDescriptor[] updatedDocs = docs.stream().map(doc -> createUpdatedDoc(doc)).toArray(DocumentDescriptor[]::new);
futures.add(updateItems(project, updatedDocs, indexer));
}
catch (Exception e) {
@@ -387,8 +384,8 @@ public class SpringSymbolIndex implements InitializingBean {
Optional<IJavaProject> maybeProject = projectFinder().find(new TextDocumentIdentifier(docURI));
if (maybeProject.isPresent()) {
try {
UpdatedDoc updatedDoc = createUpdatedDoc(docURI, content);
futures.add(updateItem(maybeProject.get(), updatedDoc, indexer));
DocumentDescriptor updatedDoc = createUpdatedDoc(docURI);
futures.add(updateItem(maybeProject.get(), updatedDoc, content, indexer));
}
catch (Exception e) {
log.error("{}", e);
@@ -417,7 +414,7 @@ public class SpringSymbolIndex implements InitializingBean {
List<String> docs = projectMapping.get(project);
try {
UpdatedDoc[] updatedDocs = docs.stream().map(doc -> createUpdatedDoc(doc, null)).toArray(UpdatedDoc[]::new);
DocumentDescriptor[] updatedDocs = docs.stream().map(doc -> createUpdatedDoc(doc)).toArray(DocumentDescriptor[]::new);
futures.add(updateItems(project, updatedDocs, indexer));
}
catch (Exception e) {
@@ -449,32 +446,16 @@ public class SpringSymbolIndex implements InitializingBean {
return docsToProject.keySet().stream().collect(Collectors.groupingBy(docURI -> docsToProject.get(docURI)));
}
private UpdatedDoc createUpdatedDoc(String docURI, String content) throws RuntimeException {
private DocumentDescriptor createUpdatedDoc(String docURI) throws RuntimeException {
try {
File file = new File(new URI(docURI));
long lastModified = file.lastModified();
Supplier<String> contentSupplier = createContentSupplier(file, content);
return new UpdatedDoc(docURI, lastModified, contentSupplier);
return new DocumentDescriptor(docURI, lastModified);
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}
private Supplier<String> createContentSupplier(File file, String content) {
return () -> {
if (content == null) {
try {
return FileUtils.readFileToString(file);
} catch (IOException e) {
log.error("{}", e);
return "";
}
} else {
return content;
}
};
}
public CompletableFuture<Void> deleteDocument(String deletedDocURI) {
synchronized(this) {
try {
@@ -665,29 +646,29 @@ public class SpringSymbolIndex implements InitializingBean {
}
}
CompletableFuture<Void> updateItem(IJavaProject project, UpdatedDoc updatedDoc, SpringIndexer indexer) {
CompletableFuture<Void> updateItem(IJavaProject project, DocumentDescriptor updatedDoc, String content, SpringIndexer indexer) {
log.debug("scheduling updateItem {}. {}, {}, {}", project.getElementName(), updatedDoc.getDocURI(), updatedDoc.getLastModified(), indexer);
return CompletableFuture.runAsync(() -> {
try {
log.debug("updateItem {}. {}, {}, {}", project.getElementName(), updatedDoc.getDocURI(), updatedDoc.getLastModified(), indexer);
indexer.updateFile(project, updatedDoc);
indexer.updateFile(project, updatedDoc, content);
} catch (Exception e) {
log.error("{}", e);
}
}, this.updateQueue);
}
CompletableFuture<Void> updateItems(IJavaProject project, UpdatedDoc[] updatedDoc, SpringIndexer indexer) {
for (UpdatedDoc doc : updatedDoc) {
CompletableFuture<Void> updateItems(IJavaProject project, DocumentDescriptor[] updatedDoc, SpringIndexer indexer) {
for (DocumentDescriptor doc : updatedDoc) {
log.debug("scheduling updateItem {}. {}, {}, {}", project.getElementName(), doc.getDocURI(), doc.getLastModified(), indexer);
}
return CompletableFuture.runAsync(() -> {
try {
for (UpdatedDoc doc : updatedDoc) {
for (DocumentDescriptor doc : updatedDoc) {
log.debug("updateItem {}. {}, {}, {}", project.getElementName(), doc.getDocURI(), doc.getLastModified(), indexer);
}

View File

@@ -10,19 +10,15 @@
*******************************************************************************/
package org.springframework.ide.vscode.boot.java.utils;
import java.util.function.Supplier;
public class UpdatedDoc {
public class DocumentDescriptor {
private final String docURI;
private final long lastModified;
private final Supplier<String> content;
public UpdatedDoc(String docURI, long lastModified, Supplier<String> content) {
public DocumentDescriptor(String docURI, long lastModified) {
super();
this.docURI = docURI;
this.lastModified = lastModified;
this.content = content;
}
public String getDocURI() {
@@ -33,8 +29,4 @@ public class UpdatedDoc {
return lastModified;
}
public Supplier<String> getContent() {
return content;
}
}

View File

@@ -23,8 +23,8 @@ public interface SpringIndexer {
void initializeProject(IJavaProject project) throws Exception;
void removeProject(IJavaProject project) throws Exception;
void updateFile(IJavaProject project, UpdatedDoc updatedDoc) throws Exception;
void updateFiles(IJavaProject project, UpdatedDoc[] updatedDocs) throws Exception;
void updateFile(IJavaProject project, DocumentDescriptor updatedDoc, String content) throws Exception;
void updateFiles(IJavaProject project, DocumentDescriptor[] updatedDocs) throws Exception;
void removeFiles(IJavaProject project, String[] docURIs) throws Exception;
}

View File

@@ -122,40 +122,22 @@ public class SpringIndexerJava implements SpringIndexer {
this.cache.remove(cacheKey);
}
/**
* Goal: collect all affected files that need to be re-scanned
* 1 - look into dependency tracker and collect all types that are contained in the initial list of files to scan
* 2 - walk through all files in dependency tracker to check which have a dependency on one of the collected types
* 3 - add them to the list of files to be re-scanned
*
* There is no recursion or loop needed anymore beyond this point, I think, since we assume
* that all changed files coming in via the initial call to the update method. There is no need to traverse the dependency
* chain. E.g.
*
* Root.java depends on Chain1.java
* Chain1.java depends on Chain2.java
*
* Chain2 comes in as a change
* -> we need to re-scan Chain1, but not Root (since Chain1 inself didn't change)
*
*/
@Override
public void updateFile(IJavaProject project, UpdatedDoc updatedDoc) throws Exception {
public void updateFile(IJavaProject project, DocumentDescriptor updatedDoc, String content) throws Exception {
SymbolCacheKey cacheKey = getCacheKey(project);
if (updatedDoc != null && shouldProcessDocument(project, updatedDoc.getDocURI())
&& isCacheOutdated(cacheKey, updatedDoc.getDocURI(), updatedDoc.getLastModified())) {
this.symbolHandler.removeSymbols(project, updatedDoc.getDocURI());
scanFile(project, updatedDoc);
scanFile(project, updatedDoc, content);
}
}
@Override
public void updateFiles(IJavaProject project, UpdatedDoc[] updatedDocs) throws Exception {
public void updateFiles(IJavaProject project, DocumentDescriptor[] updatedDocs) throws Exception {
if (updatedDocs != null) {
UpdatedDoc[] docs = filterDocuments(project, updatedDocs);
DocumentDescriptor[] docs = filterDocuments(project, updatedDocs);
for (UpdatedDoc updatedDoc : docs) {
for (DocumentDescriptor updatedDoc : docs) {
this.symbolHandler.removeSymbols(project, updatedDoc.getDocURI());
}
@@ -163,12 +145,6 @@ public class SpringIndexerJava implements SpringIndexer {
}
}
private UpdatedDoc[] filterDocuments(IJavaProject project, UpdatedDoc[] updatedDocs) {
SymbolCacheKey cacheKey = getCacheKey(project);
return Arrays.stream(updatedDocs).filter(doc -> shouldProcessDocument(project, doc.getDocURI()))
.filter(doc -> isCacheOutdated(cacheKey, doc.getDocURI(), doc.getLastModified())).toArray(UpdatedDoc[]::new);
}
@Override
public void removeFiles(IJavaProject project, String[] docURIs) throws Exception {
SymbolCacheKey cacheKey = getCacheKey(project);
@@ -179,6 +155,12 @@ public class SpringIndexerJava implements SpringIndexer {
}
}
private DocumentDescriptor[] filterDocuments(IJavaProject project, DocumentDescriptor[] updatedDocs) {
SymbolCacheKey cacheKey = getCacheKey(project);
return Arrays.stream(updatedDocs).filter(doc -> shouldProcessDocument(project, doc.getDocURI()))
.filter(doc -> isCacheOutdated(cacheKey, doc.getDocURI(), doc.getLastModified())).toArray(DocumentDescriptor[]::new);
}
private boolean shouldProcessDocument(IJavaProject project, String docURI) {
Path path = Paths.get(URI.create(docURI));
return foldersToScan(project)
@@ -192,76 +174,28 @@ public class SpringIndexerJava implements SpringIndexer {
return modifiedTimestamp > cachedModificationTImestamp;
}
private void scanFiles(IJavaProject project, UpdatedDoc[] docs) throws Exception {
ASTParser parser = createParser(project, false);
// this is to keep track of already scanned files to avoid endless loops due to circular dependencies
private void scanFiles(IJavaProject project, DocumentDescriptor[] docs) throws Exception {
Set<String> scannedFiles = new HashSet<>();
Set<String> scannedTypes = new HashSet<>();
Map<String, UpdatedDoc> updatedDocs = new HashMap<>(); // docURI -> UpdatedDoc
String[] javaFiles = new String[docs.length];
long[] lastModified = new long[docs.length];
for (int i = 0; i < docs.length; i++) {
updatedDocs.put(docs[i].getDocURI(), docs[i]);
String file = UriUtil.toFileString(docs[i].getDocURI());
javaFiles[i] = file;
lastModified[i] = docs[i].getLastModified();
scannedFiles.add(file);
}
List<CachedSymbol> generatedSymbols = new ArrayList<CachedSymbol>();
Multimap<String, String> dependencies = MultimapBuilder.hashKeys().hashSetValues().build();
FileASTRequestor requestor = new FileASTRequestor() {
@Override
public void acceptAST(String sourceFilePath, CompilationUnit cu) {
File file = new File(sourceFilePath);
String docURI = UriUtil.toUri(file).toString();
UpdatedDoc updatedDoc = updatedDocs.get(docURI);
long lastModified = updatedDoc.getLastModified();
AtomicReference<TextDocument> docRef = new AtomicReference<>();
SpringIndexerJavaContext context = new SpringIndexerJavaContext(project, cu, docURI, sourceFilePath,
lastModified, docRef, updatedDoc.getContent().get(), generatedSymbols, SCAN_PASS.ONE, new ArrayList<>(), scannedTypes);
dependencies.putAll(sourceFilePath, context.getDependencies());
scanAST(context);
fileScannedEvent(sourceFilePath);
}
};
parser.createASTs(javaFiles, null, new String[0], requestor, null);
for (CachedSymbol symbol : generatedSymbols) {
symbolHandler.addSymbol(project, symbol.getDocURI(), symbol.getEnhancedSymbol());
}
SymbolCacheKey cacheKey = getCacheKey(project);
SpringIndexerJava.this.cache.update(cacheKey, javaFiles, lastModified, generatedSymbols, dependencies);
Set<String> scannedTypes = scanFilesInternally(project, docs);
scanAffectedFiles(project, scannedTypes, scannedFiles);
}
private void scanFile(IJavaProject project, UpdatedDoc updatedDoc) throws Exception {
//TODO: optimise? Check last modified to avoid redundant scan. Reason:
// on saving a file, this may be triggered twice. Once when file is saved and once more because of a 'file changed'
// on file system. Looking at the timestamp in the cache we should be able to avoid a second scan of the exact same
// content.
private void scanFile(IJavaProject project, DocumentDescriptor updatedDoc, String content) throws Exception {
ASTParser parser = createParser(project, false);
String docURI = updatedDoc.getDocURI();
String content = updatedDoc.getContent().get();
long lastModified = updatedDoc.getLastModified();
if (content == null) {
Path path = Paths.get(new URI(docURI));
content = new String(Files.readAllBytes(path));
}
String unitName = docURI.substring(docURI.lastIndexOf("/"));
parser.setUnitName(unitName);
log.debug("Scan file: {}", unitName);
@@ -293,77 +227,91 @@ public class SpringIndexerJava implements SpringIndexer {
}
}
private void fileScannedEvent(String file) {
if (fileScanListener!=null) {
fileScanListener.fileScanned(file);
private Set<String> scanFilesInternally(IJavaProject project, DocumentDescriptor[] docs) throws Exception {
ASTParser parser = createParser(project, false);
// this is to keep track of already scanned files to avoid endless loops due to circular dependencies
Set<String> scannedTypes = new HashSet<>();
Map<String, DocumentDescriptor> updatedDocs = new HashMap<>(); // docURI -> UpdatedDoc
String[] javaFiles = new String[docs.length];
long[] lastModified = new long[docs.length];
for (int i = 0; i < docs.length; i++) {
updatedDocs.put(docs[i].getDocURI(), docs[i]);
String file = UriUtil.toFileString(docs[i].getDocURI());
javaFiles[i] = file;
lastModified[i] = docs[i].getLastModified();
}
List<CachedSymbol> generatedSymbols = new ArrayList<CachedSymbol>();
Multimap<String, String> dependencies = MultimapBuilder.hashKeys().hashSetValues().build();
FileASTRequestor requestor = new FileASTRequestor() {
@Override
public void acceptAST(String sourceFilePath, CompilationUnit cu) {
File file = new File(sourceFilePath);
String docURI = UriUtil.toUri(file).toString();
DocumentDescriptor updatedDoc = updatedDocs.get(docURI);
long lastModified = updatedDoc.getLastModified();
AtomicReference<TextDocument> docRef = new AtomicReference<>();
SpringIndexerJavaContext context = new SpringIndexerJavaContext(project, cu, docURI, sourceFilePath,
lastModified, docRef, null, generatedSymbols, SCAN_PASS.ONE, new ArrayList<>(), scannedTypes);
dependencies.putAll(sourceFilePath, context.getDependencies());
scanAST(context);
fileScannedEvent(sourceFilePath);
}
};
parser.createASTs(javaFiles, null, new String[0], requestor, null);
for (CachedSymbol symbol : generatedSymbols) {
symbolHandler.addSymbol(project, symbol.getDocURI(), symbol.getEnhancedSymbol());
}
SymbolCacheKey cacheKey = getCacheKey(project);
SpringIndexerJava.this.cache.update(cacheKey, javaFiles, lastModified, generatedSymbols, dependencies);
return scannedTypes;
}
private void scanAffectedFiles(IJavaProject project, Set<String> changedTypes, Set<String> scannedFiles) {
private void scanAffectedFiles(IJavaProject project, Set<String> changedTypes, Set<String> alreadyScannedFiles) throws Exception {
log.info("Start scanning affected files for types {}", changedTypes);
//TODO: optimise? When multiple files are 'affected', we could try to parse and scan them in batch.
// I.e. something similar to the 'scanFiles' method.
// That is probably more efficient than one by one.
Multimap<String, String> dependencies = dependencyTracker.getAllDependencies();
// Collection<String> filesToScan = new HashSet<>();
boolean scannedAnyFiles;
do {
scannedAnyFiles = false;
for (String file : dependencies.keys()) {
try {
if (!scannedFiles.contains(file)) {
Collection<String> dependsOn = dependencies.get(file);
if (dependsOn.stream().anyMatch(changedTypes::contains)) {
scannedFiles.add(file);
scannedAnyFiles = true;
log.debug("Should also scan affected file: {}", file);
File f = new File(file);
scanAffectedFile(project, UriUtil.toUri(f).toString(), f.lastModified(), FileUtils.readFileToString(f), changedTypes);
fileScannedEvent(file);
}
}
} catch (Exception e) {
log.debug("Problems scanning file {}", file, e);
}
}
if (scannedAnyFiles) {
log.debug("Some affected files where scanned, make another pass");
}
} while (scannedAnyFiles);
log.info("Finished scanning affected files {}", scannedFiles);
}
private void scanAffectedFile(IJavaProject project, String docURI, long lastModified, String content, Set<String> changedTypes) throws Exception {
symbolHandler.removeSymbols(project, docURI);
ASTParser parser = createParser(project, false);
String unitName = docURI.substring(docURI.lastIndexOf("/"));
parser.setUnitName(unitName);
parser.setSource(content.toCharArray());
CompilationUnit cu = (CompilationUnit) parser.createAST(null);
if (cu != null) {
List<CachedSymbol> generatedSymbols = new ArrayList<CachedSymbol>();
AtomicReference<TextDocument> docRef = new AtomicReference<>();
File file = UriUtil.toFile(docURI);
if (file!=null) {
SpringIndexerJavaContext context = new SpringIndexerJavaContext(
project, cu,
docURI, file.toString(), lastModified, docRef,
content, generatedSymbols,
SCAN_PASS.ONE, new ArrayList<>(), changedTypes);
scanAST(context);
SymbolCacheKey cacheKey = getCacheKey(project);
this.cache.update(cacheKey, file.getAbsolutePath(), lastModified, generatedSymbols, context.getDependencies());
// dependencyTracker.dump();
for (CachedSymbol symbol : generatedSymbols) {
symbolHandler.addSymbol(project, symbol.getDocURI(), symbol.getEnhancedSymbol());
Set<String> filesToScan = new HashSet<>();
for (String file : dependencies.keys()) {
if (!alreadyScannedFiles.contains(file)) {
Collection<String> dependsOn = dependencies.get(file);
if (dependsOn.stream().anyMatch(changedTypes::contains)) {
filesToScan.add(file);
}
}
}
DocumentDescriptor[] docsToScan = filesToScan.stream().map(file -> {
File realFile = new File(file);
String docURI = UriUtil.toUri(realFile).toString();
long lastModified = realFile.lastModified();
return new DocumentDescriptor(docURI, lastModified);
}).toArray(DocumentDescriptor[]::new);
for (DocumentDescriptor docToScan : docsToScan) {
this.symbolHandler.removeSymbols(project, docToScan.getDocURI());
}
scanFilesInternally(project, docsToScan);
log.info("Finished scanning affected files {}", alreadyScannedFiles);
}
private void scanFiles(IJavaProject project, String[] javaFiles) throws Exception {
@@ -676,7 +624,7 @@ public class SpringIndexerJava implements SpringIndexer {
File file = path.toFile();
URI docUri = UriUtil.toUri(file);
String content = FileUtils.readFileToString(file);
scanFile(project, new UpdatedDoc(docUri.toString(), file.lastModified(), () -> content));
scanFile(project, new DocumentDescriptor(docUri.toString(), file.lastModified()), content);
}
} catch (Exception e) {
log.error("{}", e);
@@ -688,4 +636,10 @@ public class SpringIndexerJava implements SpringIndexer {
this.fileScanListener = fileScanListener;
}
private void fileScannedEvent(String file) {
if (fileScanListener != null) {
fileScanListener.fileScanned(file);
}
}
}

View File

@@ -139,14 +139,14 @@ public class SpringIndexerXML implements SpringIndexer {
}
@Override
public void updateFile(IJavaProject project, UpdatedDoc updatedDoc) throws Exception {
public void updateFile(IJavaProject project, DocumentDescriptor updatedDoc, String content) throws Exception {
this.symbolHandler.removeSymbols(project, updatedDoc.getDocURI());
List<CachedSymbol> generatedSymbols = new ArrayList<CachedSymbol>();
String docURI = updatedDoc.getDocURI();
scanFile(project, updatedDoc.getContent().get(), docURI, updatedDoc.getLastModified(), generatedSymbols);
scanFile(project, content, docURI, updatedDoc.getLastModified(), generatedSymbols);
SymbolCacheKey cacheKey = getCacheKey(project);
String file = new File(new URI(docURI)).getAbsolutePath();
@@ -158,16 +158,18 @@ public class SpringIndexerXML implements SpringIndexer {
}
@Override
public void updateFiles(IJavaProject project, UpdatedDoc[] updatedDocs) throws Exception {
public void updateFiles(IJavaProject project, DocumentDescriptor[] updatedDocs) throws Exception {
List<CachedSymbol> generatedSymbols = new ArrayList<CachedSymbol>();
for (UpdatedDoc updatedDoc : updatedDocs) {
for (DocumentDescriptor updatedDoc : updatedDocs) {
String docURI = updatedDoc.getDocURI();
this.symbolHandler.removeSymbols(project, docURI);
scanFile(project, updatedDoc.getContent().get(), docURI, updatedDoc.getLastModified(), generatedSymbols);
Path path = Paths.get(new URI(docURI));
String content = new String(Files.readAllBytes(path));
scanFile(project, content, docURI, updatedDoc.getLastModified(), generatedSymbols);
SymbolCacheKey cacheKey = getCacheKey(project);
String file = new File(new URI(docURI)).getAbsolutePath();