initial change towards multi-file symbol indexing on change events

This commit is contained in:
Martin Lippert
2020-01-17 15:27:07 +01:00
parent 601006c53b
commit 6413bcb7cc
18 changed files with 722 additions and 144 deletions

View File

@@ -1,5 +1,5 @@
/*******************************************************************************
* Copyright (c) 2017, 2019 Pivotal, Inc.
* Copyright (c) 2017, 2020 Pivotal, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
@@ -13,6 +13,7 @@ package org.springframework.ide.vscode.boot.app;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -29,6 +30,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -52,6 +54,7 @@ import org.springframework.ide.vscode.boot.java.utils.SpringIndexerXMLNamespaceH
import org.springframework.ide.vscode.boot.java.utils.SymbolCache;
import org.springframework.ide.vscode.boot.java.utils.SymbolHandler;
import org.springframework.ide.vscode.boot.java.utils.SymbolIndexConfig;
import org.springframework.ide.vscode.boot.java.utils.UpdatedDoc;
import org.springframework.ide.vscode.commons.java.IJavaProject;
import org.springframework.ide.vscode.commons.java.SpringProjectUtil;
import org.springframework.ide.vscode.commons.languageserver.java.JavaProjectFinder;
@@ -65,7 +68,6 @@ import org.springframework.ide.vscode.commons.util.StringUtil;
import org.springframework.ide.vscode.commons.util.text.TextDocument;
import org.springframework.stereotype.Component;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
/**
@@ -329,17 +331,8 @@ public class SpringSymbolIndex implements InitializingBean {
if (maybeProject.isPresent()) {
try {
File file = new File(new URI(docURI));
long lastModified = file.lastModified();
Supplier<String> content = () -> {
try {
return FileUtils.readFileToString(file);
} catch (IOException e) {
log.error("{}", e);
return "";
}
};
futures.add(updateItem(maybeProject.get(), docURI, lastModified, content, indexer));
UpdatedDoc newDoc = createUpdatedDoc(docURI, null);
futures.add(updateItems(maybeProject.get(), new UpdatedDoc[] {newDoc}, indexer));
}
catch (Exception e) {
log.error("", e);
@@ -358,7 +351,8 @@ public class SpringSymbolIndex implements InitializingBean {
}
public CompletableFuture<Void> updateDocument(String docURI, String content, String reason) {
log.info("Update document [{}]: {}",reason, docURI);
log.info("Update document [{}]: {}", reason, docURI);
synchronized(this) {
List<CompletableFuture<Void>> futures = new ArrayList<>();
@@ -367,23 +361,8 @@ public class SpringSymbolIndex implements InitializingBean {
Optional<IJavaProject> maybeProject = projectFinder().find(new TextDocumentIdentifier(docURI));
if (maybeProject.isPresent()) {
try {
File file = new File(new URI(docURI));
long lastModified = file.lastModified();
Supplier<String> contentSupplier = () -> {
if (content == null) {
try {
return FileUtils.readFileToString(file);
} catch (IOException e) {
log.error("{}", e);
return "";
}
} else {
return content;
}
};
futures.add(updateItem(maybeProject.get(), docURI, lastModified, contentSupplier, indexer));
UpdatedDoc updatedDoc = createUpdatedDoc(docURI, content);
futures.add(updateItem(maybeProject.get(), updatedDoc, indexer));
}
catch (Exception e) {
log.error("{}", e);
@@ -395,6 +374,81 @@ public class SpringSymbolIndex implements InitializingBean {
}
}
public CompletableFuture<Void> updateDocuments(String[] docURIs, String reason) {
for (String docURI : docURIs) {
log.info("Update document [{}]: {}", reason, docURI);
}
synchronized(this) {
List<CompletableFuture<Void>> futures = new ArrayList<>();
for (SpringIndexer indexer : this.indexers) {
String[] interestingDocs = getDocumentsInterestingForIndexer(indexer, docURIs);
Map<String, IJavaProject> projectsForDocs = getProjectsForDocs(interestingDocs);
Map<IJavaProject, List<String>> projectMapping = getProjectMapping(projectsForDocs);
for (IJavaProject project : projectMapping.keySet()) {
List<String> docs = projectMapping.get(project);
try {
UpdatedDoc[] updatedDocs = docs.stream().map(doc -> createUpdatedDoc(doc, null)).toArray(UpdatedDoc[]::new);
futures.add(updateItems(project, updatedDocs, indexer));
}
catch (Exception e) {
log.error("{}", e);
}
}
}
return CompletableFuture.allOf((CompletableFuture[]) futures.toArray(new CompletableFuture[futures.size()]));
}
}
private Map<String, IJavaProject> getProjectsForDocs(String[] docURIs) {
Map<String, IJavaProject> result = new HashMap<>();
for (String docURI : docURIs) {
Optional<IJavaProject> project = projectFinder().find(new TextDocumentIdentifier(docURI));
if (project.isPresent()) {
result.put(docURI, project.get());
}
}
return result;
}
private String[] getDocumentsInterestingForIndexer(SpringIndexer indexer, String[] docURIs) {
return Arrays.stream(docURIs).filter(docURI -> indexer.isInterestedIn(docURI)).toArray(String[]::new);
}
private Map<IJavaProject, List<String>> getProjectMapping(Map<String, IJavaProject> docsToProject) {
return docsToProject.keySet().stream().collect(Collectors.groupingBy(docURI -> docsToProject.get(docURI)));
}
private UpdatedDoc createUpdatedDoc(String docURI, String content) throws RuntimeException {
try {
File file = new File(new URI(docURI));
long lastModified = file.lastModified();
Supplier<String> contentSupplier = createContentSupplier(file, content);
return new UpdatedDoc(docURI, lastModified, contentSupplier);
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}
private Supplier<String> createContentSupplier(File file, String content) {
return () -> {
if (content == null) {
try {
return FileUtils.readFileToString(file);
} catch (IOException e) {
log.error("{}", e);
return "";
}
} else {
return content;
}
};
}
public CompletableFuture<Void> deleteDocument(String deletedDocURI) {
synchronized(this) {
try {
@@ -561,13 +615,36 @@ public class SpringSymbolIndex implements InitializingBean {
}
}
CompletableFuture<Void> updateItem(IJavaProject project, String docURI, long lastModified, Supplier<String> content, SpringIndexer indexer) {
log.debug("scheduling updateItem {}. {}, {}, {}", project.getElementName(), docURI, lastModified, indexer);
CompletableFuture<Void> updateItem(IJavaProject project, UpdatedDoc updatedDoc, SpringIndexer indexer) {
log.debug("scheduling updateItem {}. {}, {}, {}", project.getElementName(), updatedDoc.getDocURI(), updatedDoc.getLastModified(), indexer);
return CompletableFuture.runAsync(() -> {
log.debug("updateItem {}. {}, {}, {}", project.getElementName(), docURI, lastModified, indexer);
try {
removeSymbolsByDoc(project, docURI);
indexer.updateFile(project, docURI, lastModified, content);
log.debug("updateItem {}. {}, {}, {}", project.getElementName(), updatedDoc.getDocURI(), updatedDoc.getLastModified(), indexer);
removeSymbolsByDoc(project, updatedDoc.getDocURI());
indexer.updateFile(project, updatedDoc);
} catch (Exception e) {
log.error("{}", e);
}
}, this.updateQueue);
}
CompletableFuture<Void> updateItems(IJavaProject project, UpdatedDoc[] updatedDoc, SpringIndexer indexer) {
for (UpdatedDoc doc : updatedDoc) {
log.debug("scheduling updateItem {}. {}, {}, {}", project.getElementName(), doc.getDocURI(), doc.getLastModified(), indexer);
}
return CompletableFuture.runAsync(() -> {
try {
for (UpdatedDoc doc : updatedDoc) {
log.debug("updateItem {}. {}, {}, {}", project.getElementName(), doc.getDocURI(), doc.getLastModified(), indexer);
removeSymbolsByDoc(project, doc.getDocURI());
}
indexer.updateFiles(project, updatedDoc);
} catch (Exception e) {
log.error("{}", e);
}
@@ -591,7 +668,7 @@ public class SpringSymbolIndex implements InitializingBean {
try {
removeSymbolsByDoc(project, docURI);
for (SpringIndexer index : this.indexer) {
index.removeFile(project, docURI);
index.removeFiles(project, new String[] {docURI});
}
} catch (Exception e) {
log.error("{}", e);
@@ -662,7 +739,6 @@ public class SpringSymbolIndex implements InitializingBean {
}
}
}
}
private void removeSymbolsByProject(IJavaProject project) {

View File

@@ -23,6 +23,7 @@ import com.sun.tools.attach.VirtualMachineDescriptor;
/**
* @author Martin Lippert
*/
@SuppressWarnings("restriction")
public class SpringProcessDescriptor {
private static final Logger log = LoggerFactory.getLogger(SpringProcessDescriptor.class);

View File

@@ -17,7 +17,6 @@ import java.util.Optional;
import java.util.function.Consumer;
import java.util.stream.Stream;
import org.eclipse.jdt.core.IJavaElement;
import org.eclipse.jdt.core.dom.ASTNode;
import org.eclipse.jdt.core.dom.Annotation;
import org.eclipse.jdt.core.dom.ArrayInitializer;
@@ -191,6 +190,8 @@ public class ASTUtils {
ITypeBinding klass = varBinding.getDeclaringClass();
if (klass!=null) {
dependencies.accept(klass);
}
Object constValue = varBinding.getConstantValue();
if (constValue != null) {

View File

@@ -1,5 +1,5 @@
/*******************************************************************************
* Copyright (c) 2019 Pivotal, Inc.
* Copyright (c) 2019, 2020 Pivotal, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
@@ -12,8 +12,6 @@ package org.springframework.ide.vscode.boot.java.utils;
import org.springframework.ide.vscode.commons.java.IJavaProject;
import com.google.common.base.Supplier;
/**
* @author Martin Lippert
*/
@@ -25,8 +23,8 @@ public interface SpringIndexer {
void initializeProject(IJavaProject project) throws Exception;
void removeProject(IJavaProject project) throws Exception;
void updateFile(IJavaProject project, String docURI, long lastModified, Supplier<String> content) throws Exception;
void removeFile(IJavaProject project, String docURI) throws Exception;
void updateFile(IJavaProject project, UpdatedDoc updatedDoc) throws Exception;
void updateFiles(IJavaProject project, UpdatedDoc[] updatedDocs) throws Exception;
void removeFiles(IJavaProject project, String[] docURIs) throws Exception;
}

View File

@@ -1,5 +1,5 @@
/*******************************************************************************
* Copyright (c) 2017, 2019 Pivotal, Inc.
* Copyright (c) 2017, 2020 Pivotal, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
@@ -17,7 +17,9 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -56,9 +58,7 @@ import org.springframework.ide.vscode.commons.languageserver.java.JavaProjectFin
import org.springframework.ide.vscode.commons.util.UriUtil;
import org.springframework.ide.vscode.commons.util.text.TextDocument;
import com.google.common.base.Supplier;
import com.google.common.collect.Multimap;
import com.google.common.collect.MultimapBuilder;
/**
* @author Martin Lippert
@@ -78,46 +78,8 @@ public class SpringIndexerJava implements SpringIndexer {
private boolean scanTestJavaSources = false;
private FileScanListener fileScanListener = null; //used by test code only
private final DependencyTracker dependencyTracker = new DependencyTracker();
private final SpringIndexerJavaDependencyTracker dependencyTracker = new SpringIndexerJavaDependencyTracker();
public DependencyTracker getDependencyTracker() {
return dependencyTracker;
}
public static class DependencyTracker {
private Multimap<String, String> dependencies = MultimapBuilder.hashKeys().hashSetValues().build();
public void addDependency(String sourceFile, ITypeBinding dependsOn) {
dependencies.put(sourceFile, dependsOn.getKey());
}
public void dump() {
log.info("=== Dependencies ===");
for (String sourceFile : dependencies.keySet()) {
Collection<String> values = dependencies.get(sourceFile);
if (!values.isEmpty())
log.info(sourceFile + "=> ");
for (String v : values) {
log.info(" "+v);
}
}
log.info("======================");
}
public Multimap<String, String> getAllDependencies() {
return dependencies;
}
public void update(String file, Set<String> dependenciesForFile) {
dependencies.replaceValues(file, dependenciesForFile);
}
public void restore(Multimap<String, String> deps) {
this.dependencies = deps;
}
}
public SpringIndexerJava(SymbolHandler symbolHandler, AnnotationHierarchyAwareLookup<SymbolProvider> symbolProviders, SymbolCache cache,
JavaProjectFinder projectFimder) {
this.symbolHandler = symbolHandler;
@@ -126,6 +88,10 @@ public class SpringIndexerJava implements SpringIndexer {
this.projectFinder = projectFimder;
}
public SpringIndexerJavaDependencyTracker getDependencyTracker() {
return dependencyTracker;
}
@Override
public String[] getFileWatchPatterns() {
return new String[] {"**/*.java"};
@@ -155,20 +121,53 @@ public class SpringIndexerJava implements SpringIndexer {
this.cache.remove(cacheKey);
}
/**
* Goal: collect all affected files that need to be re-scanned
* 1 - look into dependency tracker and collect all types that are contained in the initial list of files to scan
* 2 - walk through all files in dependency tracker to check which have a dependency on one of the collected types
* 3 - add them to the list of files to be re-scanned
*
* There is no recursion or loop needed anymore beyond this point, I think, since we assume
* that all changed files coming in via the initial call to the update method. There is no need to traverse the dependency
* chain. E.g.
*
* Root.java depends on Chain1.java
* Chain1.java depends on Chain2.java
*
* Chain2 comes in as a change
* -> we need to re-scan Chain1, but not Root (since Chain1 inself didn't change)
*
*/
@Override
public void updateFile(IJavaProject project, String docURI, long lastModified, Supplier<String> content) throws Exception {
if (shouldProcessDocument(project, docURI)) {
scanFile(project, docURI, lastModified, content.get());
public void updateFile(IJavaProject project, UpdatedDoc updatedDoc) throws Exception {
if (updatedDoc != null && shouldProcessDocument(project, updatedDoc.getDocURI())) {
scanFile(project, updatedDoc);
}
}
@Override
public void updateFiles(IJavaProject project, UpdatedDoc[] updatedDocs) throws Exception {
if (updatedDocs != null) {
UpdatedDoc[] docs = filterDocuments(project, updatedDocs);
scanFiles(project, docs);
}
}
private UpdatedDoc[] filterDocuments(IJavaProject project, UpdatedDoc[] updatedDocs) {
return Arrays.stream(updatedDocs).filter(doc -> shouldProcessDocument(project, doc.getDocURI())).toArray(UpdatedDoc[]::new);
}
@Override
public void removeFile(IJavaProject project, String docURI) throws Exception {
public void removeFiles(IJavaProject project, String[] docURIs) throws Exception {
SymbolCacheKey cacheKey = getCacheKey(project);
String file = new File(new URI(docURI)).getAbsolutePath();
this.cache.removeFile(cacheKey, file);
for (String docURI : docURIs) {
String file = new File(new URI(docURI)).getAbsolutePath();
this.cache.removeFile(cacheKey, file);
}
}
private boolean shouldProcessDocument(IJavaProject project, String docURI) {
Path path = Paths.get(URI.create(docURI));
return foldersToScan(project)
@@ -177,13 +176,67 @@ public class SpringIndexerJava implements SpringIndexer {
.isPresent();
}
private void scanFile(IJavaProject project, String docURI, long lastModified, String content) throws Exception {
private void scanFiles(IJavaProject project, UpdatedDoc[] docs) throws Exception {
ASTParser parser = createParser(project, false);
// this is to keep track of already scanned files to avoid endless loops due to circular dependencies
Set<String> scannedFiles = new HashSet<>();
Set<String> scannedTypes = new HashSet<>();
Map<String, UpdatedDoc> updatedDocs = new HashMap<>();
String[] javaFiles = new String[docs.length];
for (int i = 0; i < docs.length; i++) {
updatedDocs.put(docs[i].getDocURI(), docs[i]);
String file = UriUtil.toFileString(docs[i].getDocURI());
javaFiles[i] = file;
scannedFiles.add(file);
}
FileASTRequestor requestor = new FileASTRequestor() {
@Override
public void acceptAST(String sourceFilePath, CompilationUnit cu) {
File file = new File(sourceFilePath);
String docURI = UriUtil.toUri(file).toString();
long lastModified = file.lastModified();
AtomicReference<TextDocument> docRef = new AtomicReference<>();
UpdatedDoc updatedDoc = updatedDocs.get(docURI);
List<CachedSymbol> generatedSymbols = new ArrayList<CachedSymbol>();
SpringIndexerJavaContext context = new SpringIndexerJavaContext(project, cu, docURI, sourceFilePath,
lastModified, docRef, updatedDoc.getContent().get(), generatedSymbols, SCAN_PASS.ONE, new ArrayList<>(), scannedTypes);
scanAST(context);
SymbolCacheKey cacheKey = getCacheKey(project);
SpringIndexerJava.this.cache.update(cacheKey, sourceFilePath, lastModified, generatedSymbols, context.getDependencies());
for (CachedSymbol symbol : generatedSymbols) {
symbolHandler.addSymbol(project, symbol.getDocURI(), symbol.getEnhancedSymbol());
}
fileScannedEvent(sourceFilePath);
}
};
parser.createASTs(javaFiles, null, new String[0], requestor, null);
scanAffectedFiles(project, scannedTypes, scannedFiles);
}
private void scanFile(IJavaProject project, UpdatedDoc updatedDoc) throws Exception {
//TODO: optimise? Check last modified to avoid redundant scan. Reason:
// on saving a file, this may be triggered twice. Once when file is saved and once more because of a 'file changed'
// on file system. Looking at the timestamp in the cache we should be able to avoid a second scan of the exact same
// content.
ASTParser parser = createParser(project, false);
String docURI = updatedDoc.getDocURI();
String content = updatedDoc.getContent().get();
long lastModified = updatedDoc.getLastModified();
String unitName = docURI.substring(docURI.lastIndexOf("/"));
parser.setUnitName(unitName);
log.debug("Scan file: {}", unitName);
@@ -357,13 +410,7 @@ public class SpringIndexerJava implements SpringIndexer {
@Override
public boolean visit(TypeDeclaration node) {
try {
Set<String> changedTypes = context.getChangedTypes();
if (changedTypes!=null) {
ITypeBinding changedType = node.resolveBinding();
if (changedType!=null) {
changedTypes.add(changedType.getKey());
}
}
context.addScannedType(node.resolveBinding());
extractSymbolInformation(node, context);
}
catch (Exception e) {
@@ -604,7 +651,7 @@ public class SpringIndexerJava implements SpringIndexer {
File file = path.toFile();
URI docUri = UriUtil.toUri(file);
String content = FileUtils.readFileToString(file);
scanFile(project, docUri.toString(), file.lastModified(), content);
scanFile(project, new UpdatedDoc(docUri.toString(), file.lastModified(), () -> content));
}
} catch (Exception e) {
log.error("{}", e);

View File

@@ -37,7 +37,7 @@ public class SpringIndexerJavaContext {
private final SCAN_PASS pass;
private final List<String> nextPassFiles;
private final Set<String> dependencies = new HashSet<>();
private final Set<String> dependentTypes;
private final Set<String> scannedTypes;
public SpringIndexerJavaContext(
IJavaProject project,
@@ -50,7 +50,7 @@ public class SpringIndexerJavaContext {
List<CachedSymbol> generatedSymbols,
SCAN_PASS pass,
List<String> nextPassFiles,
Set<String> dependentTypes
Set<String> scannedTypes
) {
super();
this.project = project;
@@ -63,7 +63,7 @@ public class SpringIndexerJavaContext {
this.generatedSymbols = generatedSymbols;
this.pass = pass;
this.nextPassFiles = nextPassFiles;
this.dependentTypes = dependentTypes;
this.scannedTypes = scannedTypes;
}
public IJavaProject getProject() {
@@ -114,8 +114,13 @@ public class SpringIndexerJavaContext {
dependencies.add(dependsOn.getKey());
}
public Set<String> getChangedTypes() {
return dependentTypes;
public Set<String> getScannedTypes() {
return scannedTypes;
}
public void addScannedType(ITypeBinding scannedType) {
if (this.scannedTypes != null && scannedType != null) {
scannedTypes.add(scannedType.getKey());
}
}
}

View File

@@ -0,0 +1,57 @@
/*******************************************************************************
* Copyright (c) 2019, 2020 Pivotal, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* https://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Pivotal, Inc. - initial API and implementation
*******************************************************************************/
package org.springframework.ide.vscode.boot.java.utils;
import java.util.Collection;
import java.util.Set;
import org.eclipse.jdt.core.dom.ITypeBinding;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Multimap;
import com.google.common.collect.MultimapBuilder;
public class SpringIndexerJavaDependencyTracker {
private static final Logger log = LoggerFactory.getLogger(SpringIndexerJavaDependencyTracker.class);
private Multimap<String, String> dependencies = MultimapBuilder.hashKeys().hashSetValues().build();
public void addDependency(String sourceFile, ITypeBinding dependsOn) {
dependencies.put(sourceFile, dependsOn.getKey());
}
public void dump() {
log.info("=== Dependencies ===");
for (String sourceFile : dependencies.keySet()) {
Collection<String> values = dependencies.get(sourceFile);
if (!values.isEmpty())
log.info(sourceFile + "=> ");
for (String v : values) {
log.info(" "+v);
}
}
log.info("======================");
}
public Multimap<String, String> getAllDependencies() {
return dependencies;
}
public void update(String file, Set<String> dependenciesForFile) {
dependencies.replaceValues(file, dependenciesForFile);
}
public void restore(Multimap<String, String> deps) {
this.dependencies = deps;
}
}

View File

@@ -1,5 +1,5 @@
/*******************************************************************************
* Copyright (c) 2019 Pivotal, Inc.
* Copyright (c) 2019, 2020 Pivotal, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
@@ -38,8 +38,6 @@ import org.springframework.ide.vscode.commons.languageserver.java.JavaProjectFin
import org.springframework.ide.vscode.commons.util.UriUtil;
import org.springframework.ide.vscode.commons.util.text.TextDocument;
import com.google.common.base.Supplier;
/**
* @author Martin Lippert
*/
@@ -141,15 +139,16 @@ public class SpringIndexerXML implements SpringIndexer {
}
@Override
public void updateFile(IJavaProject project, String docURI, long lastModified, Supplier<String> content) throws Exception {
public void updateFile(IJavaProject project, UpdatedDoc updatedDoc) throws Exception {
List<CachedSymbol> generatedSymbols = new ArrayList<CachedSymbol>();
String docURI = updatedDoc.getDocURI();
scanFile(project, content.get(), docURI, lastModified, generatedSymbols);
scanFile(project, updatedDoc.getContent().get(), docURI, updatedDoc.getLastModified(), generatedSymbols);
SymbolCacheKey cacheKey = getCacheKey(project);
String file = new File(new URI(docURI)).getAbsolutePath();
this.cache.update(cacheKey, file, lastModified, generatedSymbols, null);
this.cache.update(cacheKey, file, updatedDoc.getLastModified(), generatedSymbols, null);
for (CachedSymbol symbol : generatedSymbols) {
symbolHandler.addSymbol(project, symbol.getDocURI(), symbol.getEnhancedSymbol());
@@ -157,10 +156,33 @@ public class SpringIndexerXML implements SpringIndexer {
}
@Override
public void removeFile(IJavaProject project, String docURI) throws Exception {
public void updateFiles(IJavaProject project, UpdatedDoc[] updatedDocs) throws Exception {
List<CachedSymbol> generatedSymbols = new ArrayList<CachedSymbol>();
for (UpdatedDoc updatedDoc : updatedDocs) {
String docURI = updatedDoc.getDocURI();
scanFile(project, updatedDoc.getContent().get(), docURI, updatedDoc.getLastModified(), generatedSymbols);
SymbolCacheKey cacheKey = getCacheKey(project);
String file = new File(new URI(docURI)).getAbsolutePath();
this.cache.update(cacheKey, file, updatedDoc.getLastModified(), generatedSymbols, null);
}
for (CachedSymbol symbol : generatedSymbols) {
symbolHandler.addSymbol(project, symbol.getDocURI(), symbol.getEnhancedSymbol());
}
}
@Override
public void removeFiles(IJavaProject project, String[] docURIs) throws Exception {
SymbolCacheKey cacheKey = getCacheKey(project);
String file = new File(new URI(docURI)).getAbsolutePath();
this.cache.removeFile(cacheKey, file);
for (String docURI : docURIs) {
String file = new File(new URI(docURI)).getAbsolutePath();
this.cache.removeFile(cacheKey, file);
}
}
private void scanFile(IJavaProject project, String fileName, List<CachedSymbol> generatedSymbols) {
@@ -254,10 +276,18 @@ public class SpringIndexerXML implements SpringIndexer {
private void clearIndex() {
for (IJavaProject project : projectFinder.all()) {
try {
for (String file : getFiles(project)) {
String docUri = UriUtil.toUri(new File(file)).toString();
symbolHandler.removeSymbols(project, docUri);
removeFile(project, docUri);
String[] files = getFiles(project);
if (files.length > 0) {
String[] docURIs = new String[files.length];
for (int i = 0; i < files.length; i++) {
String docURI = UriUtil.toUri(new File(files[i])).toString();
symbolHandler.removeSymbols(project, docURI);
docURIs[i] = docURI;
}
removeFiles(project, docURIs);
}
} catch (Exception e) {
log.error("{}", e);

View File

@@ -29,6 +29,7 @@ public interface SymbolCache {
void remove(SymbolCacheKey cacheKey);
void removeFile(SymbolCacheKey symbolCacheKey, String file);
default CachedSymbol[] retrieveSymbols(SymbolCacheKey cacheKey, String[] files) {
Pair<CachedSymbol[], Multimap<String, String>> r = retrieve(cacheKey, files);
return r!=null ? r.getLeft() : null;

View File

@@ -0,0 +1,40 @@
/*******************************************************************************
* Copyright (c) 2020 Pivotal, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* https://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Pivotal, Inc. - initial API and implementation
*******************************************************************************/
package org.springframework.ide.vscode.boot.java.utils;
import java.util.function.Supplier;
public class UpdatedDoc {
private final String docURI;
private final long lastModified;
private final Supplier<String> content;
public UpdatedDoc(String docURI, long lastModified, Supplier<String> content) {
super();
this.docURI = docURI;
this.lastModified = lastModified;
this.content = content;
}
public String getDocURI() {
return docURI;
}
public long getLastModified() {
return lastModified;
}
public Supplier<String> getContent() {
return content;
}
}

View File

@@ -88,6 +88,8 @@ public class RequestMappingDependentConstantChangedTest {
indexer.getJavaIndexer().setFileScanListener(fileScanListener);
replaceInFile(constantsUri, "path/from/constant", "/changed-path");
indexer.updateDocument(constantsUri, null, "triggered by test code").get();
fileScanListener.assertScannedUris(constantsUri, docUri);
fileScanListener.assertScannedUri(constantsUri, 1);
fileScanListener.assertScannedUri(docUri, 1);
@@ -97,7 +99,53 @@ public class RequestMappingDependentConstantChangedTest {
assertSymbol(docUri, "@/changed-path", "@RequestMapping(Constants.REQUEST_MAPPING_PATH)");
}
@Test public void testCyclicalDependency() throws Exception {
@Test
public void testSimpleRequestMappingSymbolFromConstantInDifferentClassViaMultipleFilesUpdate() throws Exception {
String docUri = directory.resolve("src/main/java/org/test/SimpleMappingClassWithConstantInDifferentClass.java").toUri().toString();
String constantsUri = directory.resolve("src/main/java/org/test/Constants.java").toUri().toString();
List<? extends SymbolInformation> symbols = indexer.getSymbols(docUri);
assertEquals(1, symbols.size());
assertSymbol(docUri, "@/path/from/constant", "@RequestMapping(Constants.REQUEST_MAPPING_PATH)");
TestFileScanListener fileScanListener = new TestFileScanListener();
indexer.getJavaIndexer().setFileScanListener(fileScanListener);
replaceInFile(constantsUri, "path/from/constant", "/changed-path");
indexer.updateDocuments(new String[] {constantsUri}, "triggered by test code").get();
fileScanListener.assertScannedUris(constantsUri, docUri);
fileScanListener.assertScannedUri(constantsUri, 1);
fileScanListener.assertScannedUri(docUri, 1);
symbols = indexer.getSymbols(docUri);
assertSymbolCount(1, symbols);
assertSymbol(docUri, "@/changed-path", "@RequestMapping(Constants.REQUEST_MAPPING_PATH)");
}
@Test
public void testRequestMappingSymbolFromConstantChained() throws Exception {
String docUri = directory.resolve("src/main/java/org/test/ChainedRequestMappingPathOverMultipleClasses.java").toUri().toString();
String chainConstantsUri_2 = directory.resolve("src/main/java/org/test/ChainElement2.java").toUri().toString();
List<? extends SymbolInformation> symbols = indexer.getSymbols(docUri);
assertEquals(1, symbols.size());
assertSymbol(docUri, "@/path/from/chain", "@RequestMapping(ChainElement1.MAPPING_PATH_1)");
replaceInFile(chainConstantsUri_2, "path/from/chain", "/changed-path");
indexer.updateDocument(chainConstantsUri_2, null, "triggered by test code").get();
symbols = indexer.getSymbols(docUri);
assertSymbolCount(1, symbols);
assertSymbol(docUri, "@/path/from/chain", "@RequestMapping(ChainElement1.MAPPING_PATH_1)");
// You would expect here that the symbol got updated from "path/from/chain" to the changed value "/changed-path",
// but the mechanism doesn't know anything about this chained dependendy. This is a limitation of the current
// implementation, since the AST has no idea about the chain, therefore we are only aware of the first
// element in this chained dependency, which comes from ChainElement1.java
}
@Test
public void testCyclicalDependency() throws Exception {
//cyclical dependency between two files (ping refers pong and vice versa)
String pingUri = directory.resolve("src/main/java/org/test/PingConstantRequestMapping.java").toUri().toString();
@@ -118,6 +166,43 @@ public class RequestMappingDependentConstantChangedTest {
}
replaceInFile(pingUri, "/ping", "/changed");
indexer.updateDocument(pingUri, null, "triggered by test code").get();
{
List<? extends SymbolInformation> symbols = indexer.getSymbols(pingUri);
assertSymbolCount(1, symbols);
assertSymbol(pingUri, "@/pong -- GET", "@GetMapping(PongConstantRequestMapping.PONG)");
}
{
List<? extends SymbolInformation> symbols = indexer.getSymbols(pongUri);
assertSymbolCount(1, symbols);
assertSymbol(pongUri, "@/changed -- GET", "@GetMapping(PingConstantRequestMapping.PING)");
}
}
@Test
public void testCyclicalDependencyViaMultipleFilesUpdate() throws Exception {
//cyclical dependency between two files (ping refers pong and vice versa)
String pingUri = directory.resolve("src/main/java/org/test/PingConstantRequestMapping.java").toUri().toString();
String pongUri = directory.resolve("src/main/java/org/test/PongConstantRequestMapping.java").toUri().toString();
{
List<? extends SymbolInformation> symbols = indexer.getSymbols(pingUri);
for (SymbolInformation s : symbols) {
System.out.println(s.getName());
}
assertSymbolCount(1, symbols);
assertSymbol(pingUri, "@/pong -- GET", "@GetMapping(PongConstantRequestMapping.PONG)");
}
{
List<? extends SymbolInformation> symbols = indexer.getSymbols(pongUri);
assertSymbolCount(1, symbols);
assertSymbol(pongUri, "@/ping -- GET", "@GetMapping(PingConstantRequestMapping.PING)");
}
replaceInFile(pingUri, "/ping", "/changed");
indexer.updateDocuments(new String[] {pingUri}, "triggered by test code").get();
{
List<? extends SymbolInformation> symbols = indexer.getSymbols(pingUri);
@@ -167,7 +252,5 @@ public class RequestMappingDependentConstantChangedTest {
assertTrue(oldContent.contains(find));
String newContent = oldContent.replace(find, replace);
FileUtils.write(target, newContent, "UTF8");
indexer.updateDocument(docUri, null, "triggered by test code").get();
}
}

View File

@@ -32,7 +32,7 @@ import org.springframework.context.annotation.Import;
import org.springframework.ide.vscode.boot.app.SpringSymbolIndex;
import org.springframework.ide.vscode.boot.bootiful.BootLanguageServerTest;
import org.springframework.ide.vscode.boot.bootiful.SymbolProviderTestConf;
import org.springframework.ide.vscode.boot.java.utils.SpringIndexerJava.DependencyTracker;
import org.springframework.ide.vscode.boot.java.utils.SpringIndexerJavaDependencyTracker;
import org.springframework.ide.vscode.commons.languageserver.java.JavaProjectFinder;
import org.springframework.ide.vscode.commons.util.UriUtil;
import org.springframework.ide.vscode.commons.util.text.LanguageId;
@@ -88,7 +88,7 @@ public class RequestMappingSymbolProviderTest {
assertTrue(containsSymbol(symbols, "@/path/from/constant", docUri, 6, 1, 6, 48));
//Verify whether dependency tracker logics works properly for this example.
DependencyTracker dt = indexer.getJavaIndexer().getDependencyTracker();
SpringIndexerJavaDependencyTracker dt = indexer.getJavaIndexer().getDependencyTracker();
assertEquals(ImmutableSet.of("Lorg/test/Constants;"), dt.getAllDependencies().get(UriUtil.toFileString(docUri)));
TestFileScanListener fileScanListener = new TestFileScanListener();

View File

@@ -0,0 +1,180 @@
/*******************************************************************************
* Copyright (c) 2020 Pivotal, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* https://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Pivotal, Inc. - initial API and implementation
*******************************************************************************/
package org.springframework.ide.vscode.boot.java.utils.test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.net.URI;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.eclipse.lsp4j.SymbolInformation;
import org.eclipse.lsp4j.TextDocumentIdentifier;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Import;
import org.springframework.ide.vscode.boot.app.SpringSymbolIndex;
import org.springframework.ide.vscode.boot.bootiful.BootLanguageServerTest;
import org.springframework.ide.vscode.boot.bootiful.SymbolProviderTestConf;
import org.springframework.ide.vscode.boot.java.utils.SymbolIndexConfig;
import org.springframework.ide.vscode.commons.java.IJavaProject;
import org.springframework.ide.vscode.commons.languageserver.java.JavaProjectFinder;
import org.springframework.ide.vscode.commons.util.Assert;
import org.springframework.ide.vscode.project.harness.BootLanguageServerHarness;
import org.springframework.ide.vscode.project.harness.ProjectsHarness;
import org.springframework.test.context.junit4.SpringRunner;
/**
* @author Martin Lippert
*/
@RunWith(SpringRunner.class)
@BootLanguageServerTest
@Import(SymbolProviderTestConf.class)
public class SpringIndexerMultipleFilesTest {
@Autowired private BootLanguageServerHarness harness;
@Autowired private SpringSymbolIndex indexer;
@Autowired private JavaProjectFinder projectFinder;
private File directory;
private String projectDir;
private IJavaProject project;
@Before
public void setup() throws Exception {
harness.intialize(null);
indexer.configureIndexer(SymbolIndexConfig.builder().scanXml(false).build());
directory = new File(ProjectsHarness.class.getResource("/test-projects/test-annotation-indexing-parent/test-annotation-indexing/").toURI());
projectDir = directory.toURI().toString();
// trigger project creation
project = projectFinder.find(new TextDocumentIdentifier(projectDir)).get();
CompletableFuture<Void> initProject = indexer.waitOperation();
initProject.get(5, TimeUnit.SECONDS);
}
@Test
public void testUpdateChangedSingleDocumentOnDisc() throws Exception {
String changedDocURI = directory.toPath().resolve("src/main/java/org/test/SimpleMappingClass.java").toUri().toString();
String originalContent = FileUtils.readFileToString(new File(new URI(changedDocURI)));
try {
// update document and update index
assertTrue(containsSymbol(indexer.getSymbols(changedDocURI), "@/mapping1", changedDocURI));
String newContent = originalContent.replace("mapping1", "mapping1-CHANGED");
FileUtils.writeStringToFile(new File(new URI(changedDocURI)), newContent);
CompletableFuture<Void> updateFuture = indexer.updateDocument(changedDocURI, null, "test triggered");
updateFuture.get(5, TimeUnit.SECONDS);
// check for updated index per document
List<? extends SymbolInformation> symbols = indexer.getSymbols(changedDocURI);
assertEquals(2, symbols.size());
assertTrue(containsSymbol(symbols, "@/mapping1-CHANGED", changedDocURI, 6, 1, 6, 36));
assertTrue(containsSymbol(symbols, "@/mapping2", changedDocURI, 11, 1, 11, 28));
}
finally {
FileUtils.writeStringToFile(new File(new URI(changedDocURI)), originalContent);
}
}
@Test
public void testUpdateChangedMultipleDocumentsOnDisc() throws Exception {
String doc1URI = directory.toPath().resolve("src/main/java/org/test/SimpleMappingClass.java").toUri().toString();
String original1Content = FileUtils.readFileToString(new File(new URI(doc1URI)));
String doc2URI = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString();
String original2Content = FileUtils.readFileToString(new File(new URI(doc2URI)));
String doc3URI = directory.toPath().resolve("src/main/java/org/test/sub/MappingClassSubpackage.java").toUri().toString();
String original3Content = FileUtils.readFileToString(new File(new URI(doc3URI)));
try {
String new1Content = original1Content.replace("mapping1", "mapping1-CHANGED");
FileUtils.writeStringToFile(new File(new URI(doc1URI)), new1Content);
String new2Content = original2Content.replace("\"/embedded-foo-mapping\"", "\"/embedded-foo-mapping-CHANGED\"");
FileUtils.writeStringToFile(new File(new URI(doc2URI)), new2Content);
String new3Content = original3Content.replace("classlevel", "classlevel-CHANGED");
FileUtils.writeStringToFile(new File(new URI(doc3URI)), new3Content);
CompletableFuture<Void> updateFuture = indexer.updateDocuments(new String[] {doc1URI, doc2URI, doc3URI}, "test triggered");
updateFuture.get(5, TimeUnit.SECONDS);
// check for updated index per document
List<? extends SymbolInformation> symbols1 = indexer.getSymbols(doc1URI);
assertEquals(2, symbols1.size());
assertTrue(containsSymbol(symbols1, "@/mapping1-CHANGED", doc1URI, 6, 1, 6, 36));
assertTrue(containsSymbol(symbols1, "@/mapping2", doc1URI, 11, 1, 11, 28));
List<? extends SymbolInformation> symbols2 = indexer.getSymbols(doc2URI);
assertTrue(containsSymbol(symbols2, "@+ 'mainClass' (@SpringBootApplication <: @SpringBootConfiguration, @Configuration, @Component) MainClass", doc2URI, 6, 0, 6, 22));
assertTrue(containsSymbol(symbols2, "@/embedded-foo-mapping-CHANGED", doc2URI, 17, 1, 17, 49));
assertTrue(containsSymbol(symbols2, "@/foo-root-mapping/embedded-foo-mapping-with-root", doc2URI, 27, 1, 27, 51));
List<? extends SymbolInformation> symbols3 = indexer.getSymbols(doc3URI);
assertTrue(containsSymbol(symbols3, "@/classlevel-CHANGED/mapping-subpackage", doc3URI, 7, 1, 7, 38));
}
finally {
FileUtils.writeStringToFile(new File(new URI(doc1URI)), original1Content);
FileUtils.writeStringToFile(new File(new URI(doc2URI)), original2Content);
FileUtils.writeStringToFile(new File(new URI(doc3URI)), original3Content);
}
}
private boolean containsSymbol(List<? extends SymbolInformation> symbols, String name, String uri) {
for (Iterator<? extends SymbolInformation> iterator = symbols.iterator(); iterator.hasNext();) {
SymbolInformation symbol = iterator.next();
if (
symbol.getName().equals(name) &&
symbol.getLocation().getUri().equals(uri)
) {
return true;
}
}
return false;
}
private boolean containsSymbol(List<? extends SymbolInformation> symbols, String name, String uri, int startLine, int startCHaracter, int endLine, int endCharacter) {
for (Iterator<? extends SymbolInformation> iterator = symbols.iterator(); iterator.hasNext();) {
SymbolInformation symbol = iterator.next();
if (symbol.getName().equals(name)
&& symbol.getLocation().getUri().equals(uri)
&& symbol.getLocation().getRange().getStart().getLine() == startLine
&& symbol.getLocation().getRange().getStart().getCharacter() == startCHaracter
&& symbol.getLocation().getRange().getEnd().getLine() == endLine
&& symbol.getLocation().getRange().getEnd().getCharacter() == endCharacter) {
return true;
}
}
return false;
}
}

View File

@@ -1,22 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="https://maven.apache.org/POM/4.0.0" xmlns:xsi="https://www.w3.org/2001/XMLSchema-instance"
<project xmlns="https://maven.apache.org/POM/4.0.0"
xmlns:xsi="https://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="https://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>test-annotation-indexing-non-boot-project</artifactId>
<groupId>test-projects</groupId>
<version>5.1.3</version>
<groupId>test-projects</groupId>
<version>5.1.3</version>
<packaging>jar</packaging>
<name>test-annotation-indexing-non-boot-project</name>
<description>Test projects for regular non-boot spring project - annotation indexing</description>
<dependencies>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc</artifactId>
<version>5.1.3.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc</artifactId>
<version>5.1.3.RELEASE</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
</project>

View File

@@ -1,22 +1,41 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="https://maven.apache.org/POM/4.0.0" xmlns:xsi="https://www.w3.org/2001/XMLSchema-instance"
<project xmlns="https://maven.apache.org/POM/4.0.0"
xmlns:xsi="https://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="https://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>test-annotation-indexing-non-boot-project</artifactId>
<groupId>test-projects</groupId>
<version>5.1.3</version>
<groupId>test-projects</groupId>
<version>5.1.3</version>
<packaging>jar</packaging>
<name>test-annotation-indexing-non-boot-project</name>
<description>Test projects for regular non-boot spring project - annotation indexing</description>
<properties>
<maven.javadoc.skip>true</maven.javadoc.skip>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc</artifactId>
<version>5.1.3.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc</artifactId>
<version>5.1.3.RELEASE</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
</project>

View File

@@ -0,0 +1,7 @@
package org.test;
public class ChainElement1 {
public static final String MAPPING_PATH_1 = ChainElement2.MAPPING_PATH_2;
}

View File

@@ -0,0 +1,7 @@
package org.test;
public class ChainElement2 {
public static final String MAPPING_PATH_2 = "path/from/chain";
}

View File

@@ -0,0 +1,12 @@
package org.test;
import org.springframework.web.bind.annotation.RequestMapping;
public class ChainedRequestMappingOverMultipleClasses {
@RequestMapping(ChainElement1.MAPPING_PATH_1)
public String hello() {
return "Hello";
}
}