optimize java symbol indexer performance by avoiding double scanning of the same file in case the timestamp has not changed

This commit is contained in:
Martin Lippert
2020-02-06 10:16:22 +01:00
parent 83e84e5863
commit fbb306b9ad
11 changed files with 243 additions and 27 deletions

View File

@@ -142,7 +142,9 @@ public class SpringIndexerJava implements SpringIndexer {
@Override
public void updateFile(IJavaProject project, UpdatedDoc updatedDoc) throws Exception {
if (updatedDoc != null && shouldProcessDocument(project, updatedDoc.getDocURI())) {
SymbolCacheKey cacheKey = getCacheKey(project);
if (updatedDoc != null && shouldProcessDocument(project, updatedDoc.getDocURI())
&& hasNewerModificationTimestamp(cacheKey, updatedDoc.getDocURI(), updatedDoc.getLastModified())) {
scanFile(project, updatedDoc);
}
}
@@ -156,7 +158,9 @@ public class SpringIndexerJava implements SpringIndexer {
}
private UpdatedDoc[] filterDocuments(IJavaProject project, UpdatedDoc[] updatedDocs) {
return Arrays.stream(updatedDocs).filter(doc -> shouldProcessDocument(project, doc.getDocURI())).toArray(UpdatedDoc[]::new);
SymbolCacheKey cacheKey = getCacheKey(project);
return Arrays.stream(updatedDocs).filter(doc -> shouldProcessDocument(project, doc.getDocURI()))
.filter(doc -> hasNewerModificationTimestamp(cacheKey, doc.getDocURI(), doc.getLastModified())).toArray(UpdatedDoc[]::new);
}
@Override
@@ -176,6 +180,11 @@ public class SpringIndexerJava implements SpringIndexer {
.findFirst()
.isPresent();
}
private boolean hasNewerModificationTimestamp(SymbolCacheKey cacheKey, String docURI, long modifiedTimestamp) {
long cachedModificationTImestamp = this.cache.getModificationTimestamp(cacheKey, UriUtil.toFileString(docURI));
return modifiedTimestamp > cachedModificationTImestamp;
}
private void scanFiles(IJavaProject project, UpdatedDoc[] docs) throws Exception {
ASTParser parser = createParser(project, false);

View File

@@ -35,5 +35,7 @@ public interface SymbolCache {
Pair<CachedSymbol[], Multimap<String, String>> r = retrieve(cacheKey, files);
return r!=null ? r.getLeft() : null;
}
long getModificationTimestamp(SymbolCacheKey cacheKey, String docURI);
}

View File

@@ -241,6 +241,20 @@ public class SymbolCacheOnDisc implements SymbolCache {
}
}
@Override
public long getModificationTimestamp(SymbolCacheKey cacheKey, String file) {
CacheStore cacheStore = this.stores.get(cacheKey);
if (cacheStore != null) {
Long result = cacheStore.getTimestampedFiles().get(file);
if (result != null) {
return result;
}
}
return 0;
}
private void save(SymbolCacheKey cacheKey, List<CachedSymbol> generatedSymbols,
SortedMap<String, Long> timestampedFiles, Map<String, Collection<String>> dependencies) {
CacheStore store = new CacheStore(timestampedFiles, generatedSymbols, dependencies);

View File

@@ -47,5 +47,9 @@ public class SymbolCacheVoid implements SymbolCache {
public void removeFile(SymbolCacheKey symbolCacheKey, String file) {
}
@Override
public long getModificationTimestamp(SymbolCacheKey cacheKey, String docURI) {
return 0;
}
}

View File

@@ -33,6 +33,7 @@ import org.springframework.context.annotation.Import;
import org.springframework.ide.vscode.boot.app.SpringSymbolIndex;
import org.springframework.ide.vscode.boot.bootiful.BootLanguageServerTest;
import org.springframework.ide.vscode.boot.bootiful.SymbolProviderTestConf;
import org.springframework.ide.vscode.boot.java.utils.test.TestFileScanListener;
import org.springframework.ide.vscode.commons.languageserver.java.JavaProjectFinder;
import org.springframework.ide.vscode.commons.maven.java.MavenJavaProject;
import org.springframework.ide.vscode.commons.util.UriUtil;

View File

@@ -1,5 +1,5 @@
/*******************************************************************************
* Copyright (c) 2017, 2019 Pivotal, Inc.
* Copyright (c) 2017, 2020 Pivotal, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
@@ -33,6 +33,7 @@ import org.springframework.ide.vscode.boot.app.SpringSymbolIndex;
import org.springframework.ide.vscode.boot.bootiful.BootLanguageServerTest;
import org.springframework.ide.vscode.boot.bootiful.SymbolProviderTestConf;
import org.springframework.ide.vscode.boot.java.utils.SpringIndexerJavaDependencyTracker;
import org.springframework.ide.vscode.boot.java.utils.test.TestFileScanListener;
import org.springframework.ide.vscode.commons.languageserver.java.JavaProjectFinder;
import org.springframework.ide.vscode.commons.util.UriUtil;
import org.springframework.ide.vscode.commons.util.text.LanguageId;
@@ -50,7 +51,7 @@ import com.google.common.collect.ImmutableSet;
@BootLanguageServerTest
@Import(SymbolProviderTestConf.class)
public class RequestMappingSymbolProviderTest {
@Autowired private BootLanguageServerHarness harness;
@Autowired private JavaProjectFinder projectFinder;
@Autowired private SpringSymbolIndex indexer;
@@ -93,7 +94,10 @@ public class RequestMappingSymbolProviderTest {
TestFileScanListener fileScanListener = new TestFileScanListener();
indexer.getJavaIndexer().setFileScanListener(fileScanListener);
indexer.updateDocument(constantsUri, FileUtils.readFileToString(UriUtil.toFile(constantsUri)), "test triggered").get();
CompletableFuture<Void> updateFuture = indexer.updateDocument(constantsUri, FileUtils.readFileToString(UriUtil.toFile(constantsUri)), "test triggered");
updateFuture.get(5, TimeUnit.SECONDS);
fileScanListener.assertScannedUris(constantsUri, docUri);
fileScanListener.assertScannedUri(constantsUri, 1);
fileScanListener.assertScannedUri(docUri, 1);
@@ -113,12 +117,18 @@ public class RequestMappingSymbolProviderTest {
TestFileScanListener fileScanListener = new TestFileScanListener();
indexer.getJavaIndexer().setFileScanListener(fileScanListener);
indexer.updateDocument(pingUri, null, "test triggered").get();
CompletableFuture<Void> updateFuture = indexer.updateDocument(pingUri, null, "test triggered");
updateFuture.get(5, TimeUnit.SECONDS);
fileScanListener.assertScannedUris(pingUri, pongUri);
fileScanListener.reset();
fileScanListener.assertScannedUris(/*none*/);
indexer.updateDocument(pongUri, null, "test triggered").get();
CompletableFuture<Void> updateFuture2 = indexer.updateDocument(pongUri, null, "test triggered");
updateFuture2.get(5, TimeUnit.SECONDS);
fileScanListener.assertScannedUris(pingUri, pongUri);
}

View File

@@ -11,8 +11,6 @@
package org.springframework.ide.vscode.boot.java.utils.test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.File;
@@ -29,14 +27,15 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.ide.vscode.boot.app.SpringSymbolIndex;
import org.springframework.ide.vscode.boot.bootiful.BootLanguageServerTest;
import org.springframework.ide.vscode.boot.bootiful.SymbolProviderTestConf;
import org.springframework.ide.vscode.boot.java.utils.SymbolCache;
import org.springframework.ide.vscode.boot.java.utils.SymbolIndexConfig;
import org.springframework.ide.vscode.commons.java.IJavaProject;
import org.springframework.ide.vscode.commons.languageserver.java.JavaProjectFinder;
import org.springframework.ide.vscode.commons.util.Assert;
import org.springframework.ide.vscode.project.harness.BootLanguageServerHarness;
import org.springframework.ide.vscode.project.harness.ProjectsHarness;
import org.springframework.test.context.junit4.SpringRunner;
@@ -46,16 +45,25 @@ import org.springframework.test.context.junit4.SpringRunner;
*/
@RunWith(SpringRunner.class)
@BootLanguageServerTest
@Import(SymbolProviderTestConf.class)
@Import(SpringIndexerMultipleFilesTest.TimestampingAwareCacheConfig.class)
public class SpringIndexerMultipleFilesTest {
// usually, the test config ignores any caching by using the void impl,
// but here we need the one that implements at least the timestamp caching
// in order to check the java symbol indexer feature which avoid scanning the
// same file again even if the timestamp hasn't changed
public static class TimestampingAwareCacheConfig extends SymbolProviderTestConf {
@Bean public SymbolCache symbolCache() {
return new SymbolCacheTimestampsOnly();
}
}
@Autowired private BootLanguageServerHarness harness;
@Autowired private SpringSymbolIndex indexer;
@Autowired private JavaProjectFinder projectFinder;
private File directory;
private String projectDir;
private IJavaProject project;
@Before
public void setup() throws Exception {
@@ -66,7 +74,7 @@ public class SpringIndexerMultipleFilesTest {
projectDir = directory.toURI().toString();
// trigger project creation
project = projectFinder.find(new TextDocumentIdentifier(projectDir)).get();
projectFinder.find(new TextDocumentIdentifier(projectDir)).get();
CompletableFuture<Void> initProject = indexer.waitOperation();
initProject.get(5, TimeUnit.SECONDS);
@@ -74,25 +82,31 @@ public class SpringIndexerMultipleFilesTest {
@Test
public void testUpdateChangedSingleDocumentOnDisc() throws Exception {
String changedDocURI = directory.toPath().resolve("src/main/java/org/test/SimpleMappingClass.java").toUri().toString();
String originalContent = FileUtils.readFileToString(new File(new URI(changedDocURI)));
try {
// update document and update index
assertTrue(containsSymbol(indexer.getSymbols(changedDocURI), "@/mapping1", changedDocURI));
List<? extends SymbolInformation> symbols = indexer.getSymbols(changedDocURI);
assertTrue(containsSymbol(symbols, "@/mapping1", changedDocURI));
String newContent = originalContent.replace("mapping1", "mapping1-CHANGED");
FileUtils.writeStringToFile(new File(new URI(changedDocURI)), newContent);
TestFileScanListener fileScanListener = new TestFileScanListener();
indexer.getJavaIndexer().setFileScanListener(fileScanListener);
CompletableFuture<Void> updateFuture = indexer.updateDocument(changedDocURI, null, "test triggered");
updateFuture.get(5, TimeUnit.SECONDS);
// check for updated index per document
List<? extends SymbolInformation> symbols = indexer.getSymbols(changedDocURI);
symbols = indexer.getSymbols(changedDocURI);
assertEquals(2, symbols.size());
assertTrue(containsSymbol(symbols, "@/mapping1-CHANGED", changedDocURI, 6, 1, 6, 36));
assertTrue(containsSymbol(symbols, "@/mapping2", changedDocURI, 11, 1, 11, 28));
fileScanListener.assertScannedUris(changedDocURI);
fileScanListener.assertScannedUri(changedDocURI, 1);
}
finally {
FileUtils.writeStringToFile(new File(new URI(changedDocURI)), originalContent);
@@ -144,6 +158,64 @@ public class SpringIndexerMultipleFilesTest {
FileUtils.writeStringToFile(new File(new URI(doc3URI)), original3Content);
}
}
@Test
public void testDontScanUnchangedDocument() throws Exception {
String unchangedDocURI = directory.toPath().resolve("src/main/java/org/test/SimpleMappingClass.java").toUri().toString();
TestFileScanListener fileScanListener = new TestFileScanListener();
indexer.getJavaIndexer().setFileScanListener(fileScanListener);
CompletableFuture<Void> updateFuture = indexer.updateDocuments(new String[] {unchangedDocURI}, "test triggered");
updateFuture.get(5, TimeUnit.SECONDS);
fileScanListener.assertScannedUris();
fileScanListener.assertScannedUri(unchangedDocURI, 0);
}
@Test
public void testDontScanUnchangedDocumentAmongMultipleChangedFiles() throws Exception {
String doc1URI = directory.toPath().resolve("src/main/java/org/test/SimpleMappingClass.java").toUri().toString();
String original1Content = FileUtils.readFileToString(new File(new URI(doc1URI)));
String doc2URI = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString();
String doc3URI = directory.toPath().resolve("src/main/java/org/test/sub/MappingClassSubpackage.java").toUri().toString();
String original3Content = FileUtils.readFileToString(new File(new URI(doc3URI)));
try {
String new1Content = original1Content.replace("mapping1", "mapping1-CHANGED");
FileUtils.writeStringToFile(new File(new URI(doc1URI)), new1Content);
String new3Content = original3Content.replace("classlevel", "classlevel-CHANGED");
FileUtils.writeStringToFile(new File(new URI(doc3URI)), new3Content);
TestFileScanListener fileScanListener = new TestFileScanListener();
indexer.getJavaIndexer().setFileScanListener(fileScanListener);
CompletableFuture<Void> updateFuture = indexer.updateDocuments(new String[] {doc1URI, doc2URI, doc3URI}, "test triggered");
updateFuture.get(5, TimeUnit.SECONDS);
// check for updated index per document
List<? extends SymbolInformation> symbols1 = indexer.getSymbols(doc1URI);
assertEquals(2, symbols1.size());
assertTrue(containsSymbol(symbols1, "@/mapping1-CHANGED", doc1URI, 6, 1, 6, 36));
assertTrue(containsSymbol(symbols1, "@/mapping2", doc1URI, 11, 1, 11, 28));
List<? extends SymbolInformation> symbols3 = indexer.getSymbols(doc3URI);
assertTrue(containsSymbol(symbols3, "@/classlevel-CHANGED/mapping-subpackage", doc3URI, 7, 1, 7, 38));
fileScanListener.assertScannedUris(doc1URI, doc3URI);
fileScanListener.assertScannedUri(doc1URI, 1);
fileScanListener.assertScannedUri(doc2URI, 0);
fileScanListener.assertScannedUri(doc3URI, 1);
}
finally {
FileUtils.writeStringToFile(new File(new URI(doc1URI)), original1Content);
FileUtils.writeStringToFile(new File(new URI(doc3URI)), original3Content);
}
}
private boolean containsSymbol(List<? extends SymbolInformation> symbols, String name, String uri) {
for (Iterator<? extends SymbolInformation> iterator = symbols.iterator(); iterator.hasNext();) {

View File

@@ -1,5 +1,5 @@
/*******************************************************************************
* Copyright (c) 2018 Pivotal, Inc.
* Copyright (c) 2018, 2020 Pivotal, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
@@ -29,7 +29,6 @@ import org.springframework.context.annotation.Import;
import org.springframework.ide.vscode.boot.app.SpringSymbolIndex;
import org.springframework.ide.vscode.boot.bootiful.BootLanguageServerTest;
import org.springframework.ide.vscode.boot.bootiful.SymbolProviderTestConf;
import org.springframework.ide.vscode.commons.java.IJavaProject;
import org.springframework.ide.vscode.commons.languageserver.java.JavaProjectFinder;
import org.springframework.ide.vscode.project.harness.BootLanguageServerHarness;
import org.springframework.ide.vscode.project.harness.ProjectsHarness;
@@ -49,7 +48,6 @@ public class SpringIndexerNonBootProjectTest {
private File directory;
private String projectDir;
private IJavaProject project;
@Before
public void setup() throws Exception {
@@ -59,7 +57,7 @@ public class SpringIndexerNonBootProjectTest {
projectDir = directory.toURI().toString();
// trigger project creation
project = projectFinder.find(new TextDocumentIdentifier(projectDir)).get();
projectFinder.find(new TextDocumentIdentifier(projectDir)).get();
CompletableFuture<Void> initProject = indexer.waitOperation();
initProject.get(5, TimeUnit.SECONDS);

View File

@@ -109,6 +109,9 @@ public class SymbolCacheOnDiscTest {
assertEquals(2, dependencies.keySet().size());
assertEquals(dependencies.get(file1.toString()), ImmutableSet.of("file1dep1"));
assertEquals(dependencies.get(file2.toString()), ImmutableSet.of("file2dep1", "file2dep2"));
assertEquals(timeFile1.toMillis(), cache.getModificationTimestamp(new SymbolCacheKey("somekey", "1"), file1.toString()));
assertEquals(0, cache.getModificationTimestamp(new SymbolCacheKey("somekey", "1"), "random-non-existing-file"));
}
@Test
@@ -282,6 +285,8 @@ public class SymbolCacheOnDiscTest {
CachedSymbol[] cachedSymbols = cache.retrieveSymbols(new SymbolCacheKey("somekey", "1"), files);
assertNotNull(cachedSymbols);
assertEquals(2, cachedSymbols.length);
assertEquals(timeFile1.toMillis() + 2000, cache.getModificationTimestamp(new SymbolCacheKey("somekey", "1"), file1.toString()));
}
@Test
@@ -356,6 +361,10 @@ public class SymbolCacheOnDiscTest {
assertSymbol(newEnhancedSymbol1, cachedSymbols);
assertSymbol(updatedEnhancedSymbol2, cachedSymbols);
assertSymbol(enhancedSymbol3, cachedSymbols);
assertEquals(timeFile1.toMillis() + 2000, cache.getModificationTimestamp(new SymbolCacheKey("somekey", "1"), file1.toString()));
assertEquals(timeFile2.toMillis() + 3000, cache.getModificationTimestamp(new SymbolCacheKey("somekey", "1"), file2.toString()));
assertEquals(timeFile3.toMillis(), cache.getModificationTimestamp(new SymbolCacheKey("somekey", "1"), file3.toString()));
}
private void assertSymbol(EnhancedSymbolInformation enhancedSymbol, CachedSymbol[] cachedSymbols) {
@@ -494,13 +503,9 @@ public class SymbolCacheOnDiscTest {
Files.createFile(file1);
Files.createFile(file2);
FileTime timeFile1 = Files.getLastModifiedTime(file1);
FileTime timeFile2 = Files.getLastModifiedTime(file2);
String[] files = {file1.toString()};
String doc1URI = UriUtil.toUri(file1.toFile()).toString();
String doc2URI = UriUtil.toUri(file2.toFile()).toString();
List<CachedSymbol> generatedSymbols1 = ImmutableList.of();
Multimap<String, String> dependencies1 = ImmutableMultimap.of(
file1.toString(), "dep1"

View File

@@ -0,0 +1,101 @@
/*******************************************************************************
* Copyright (c) 2020 Pivotal, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* https://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Pivotal, Inc. - initial API and implementation
*******************************************************************************/
package org.springframework.ide.vscode.boot.java.utils.test;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.ide.vscode.boot.java.utils.CachedSymbol;
import org.springframework.ide.vscode.boot.java.utils.SymbolCache;
import org.springframework.ide.vscode.boot.java.utils.SymbolCacheKey;
import com.google.common.collect.Multimap;
/**
* @author Martin Lippert
*/
public class SymbolCacheTimestampsOnly implements SymbolCache {
private Map<SymbolCacheKey, Map<String, Long>> timestampCache;
public SymbolCacheTimestampsOnly() {
this.timestampCache = new HashMap<>();
}
@Override
public void store(SymbolCacheKey cacheKey, String[] files, List<CachedSymbol> generatedSymbols, Multimap<String,String> dependencies) {
SortedMap<String, Long> timestampedFiles = new TreeMap<>();
timestampedFiles = Arrays.stream(files)
.filter(file -> new File(file).exists())
.collect(Collectors.toMap(file -> file, file -> {
try {
return Files.getLastModifiedTime(new File(file).toPath()).toMillis();
} catch (IOException e) {
throw new RuntimeException(e);
}
}, (v1,v2) -> { throw new RuntimeException(String.format("Duplicate key for values %s and %s", v1, v2));}, TreeMap::new));
timestampCache.put(cacheKey, timestampedFiles);
}
@Override
public Pair<CachedSymbol[], Multimap<String, String>> retrieve(SymbolCacheKey cacheKey, String[] files) {
return null;
}
@Override
public void update(SymbolCacheKey cacheKey, String file, long lastModified, List<CachedSymbol> generatedSymbols, Set<String> dependencies) {
Map<String, Long> timestampMap = timestampCache.get(cacheKey);
timestampMap.put(file, lastModified);
}
@Override
public void update(SymbolCacheKey cacheKey, String[] files, long[] lastModified, List<CachedSymbol> generatedSymbols, Multimap<String, String> dependencies) {
Map<String, Long> timestampMap = timestampCache.get(cacheKey);
for (int i = 0; i < files.length; i++) {
timestampMap.put(files[i], lastModified[i]);
}
}
@Override
public void remove(SymbolCacheKey cacheKey) {
}
@Override
public void removeFile(SymbolCacheKey symbolCacheKey, String file) {
}
@Override
public long getModificationTimestamp(SymbolCacheKey cacheKey, String file) {
Map<String, Long> timestampMap = timestampCache.get(cacheKey);
if (timestampMap != null) {
Long timestamp = timestampMap.get(file);
if (timestamp != null) {
return timestamp;
}
}
return 0;
}
}

View File

@@ -1,5 +1,5 @@
/*******************************************************************************
* Copyright (c) 2019 Pivotal, Inc.
* Copyright (c) 2019, 2020 Pivotal, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
@@ -8,7 +8,7 @@
* Contributors:
* Pivotal, Inc. - initial API and implementation
*******************************************************************************/
package org.springframework.ide.vscode.boot.java.requestmapping.test;
package org.springframework.ide.vscode.boot.java.utils.test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;