Spring/spring-tools
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-1219: java indexer now splits set of java source files to scan into smaller chunks to reduceo overall memory needs

Browse Source
This commit is contained in:
Martin Lippert
2024-04-03 16:02:57 +02:00
parent 0477995c61
commit 0efe8cd63a
12 changed files with 207 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/utils/SpringIndexerJava.java
View File

@@ -59,8 +59,8 @@ import org.springframework.ide.vscode.boot.java.beans.CachedBean;
import org.springframework.ide.vscode.boot.java.handlers.EnhancedSymbolInformation;
import org.springframework.ide.vscode.boot.java.handlers.SymbolProvider;
import org.springframework.ide.vscode.boot.java.reconcilers.CachedDiagnostics;
import org.springframework.ide.vscode.boot.java.reconcilers.RequiredCompleteAstException;
import org.springframework.ide.vscode.boot.java.reconcilers.JdtReconciler;
import org.springframework.ide.vscode.boot.java.reconcilers.RequiredCompleteAstException;
import org.springframework.ide.vscode.commons.java.IClasspath;
import org.springframework.ide.vscode.commons.java.IClasspathUtil;
import org.springframework.ide.vscode.commons.java.IJavaProject;
@@ -108,6 +108,7 @@ public class SpringIndexerJava implements SpringIndexer {
private boolean scanTestJavaSources = false;
private JsonObject validationSeveritySettings;
private int scanChunkSize = 1000;
private FileScanListener fileScanListener = null; //used by test code only
@@ -362,7 +363,6 @@ public class SpringIndexerJava implements SpringIndexer {
private Set<String> scanFilesInternally(IJavaProject project, DocumentDescriptor[] docs) throws Exception {
final boolean ignoreMethodBodies = false;
ASTParser parser = createParser(project, ignoreMethodBodies);
// this is to keep track of already scanned files to avoid endless loops due to circular dependencies
Set<String> scannedTypes = new HashSet<>();
@@ -418,7 +418,11 @@ public class SpringIndexerJava implements SpringIndexer {
}
};
parser.createASTs(javaFiles, null, new String[0], requestor, null);
List<String[]> chunks = createChunks(javaFiles, this.scanChunkSize);
for(int i = 0; i < chunks.size(); i++) {
ASTParser parser = createParser(project, ignoreMethodBodies);
parser.createASTs(chunks.get(i), null, new String[0], requestor, null);
}
EnhancedSymbolInformation[] symbols = generatedSymbols.stream().map(cachedSymbol -> cachedSymbol.getEnhancedSymbol()).toArray(EnhancedSymbolInformation[]::new);
Bean[] beans = generatedBeans.stream().filter(cachedBean -> cachedBean.getBean() != null).map(cachedBean -> cachedBean.getBean()).toArray(Bean[]::new);
@@ -497,14 +501,18 @@ public class SpringIndexerJava implements SpringIndexer {
}
};
String[] pass2Files = scanFiles(project, javaFiles, generatedSymbols, generatedBeans, diagnosticsAggregator, SCAN_PASS.ONE);
if (pass2Files.length > 0) {
List<String[]> chunks = createChunks(javaFiles, this.scanChunkSize);
for (int i = 0; i < chunks.size(); i++) {
log.info("scan java files, AST parse, pass 2 for files: {}", javaFiles.length);
scanFiles(project, pass2Files, generatedSymbols, generatedBeans, diagnosticsAggregator, SCAN_PASS.TWO);
}
log.info("scan java files, AST parse, chunk {} for files: {}", i, javaFiles.length);
String[] pass2Files = scanFiles(project, chunks.get(i), generatedSymbols, generatedBeans, diagnosticsAggregator, SCAN_PASS.ONE);
if (pass2Files.length > 0) {
log.info("scan java files, AST parse, pass 2, chunk {} for files: {}", i, javaFiles.length);
scanFiles(project, pass2Files, generatedSymbols, generatedBeans, diagnosticsAggregator, SCAN_PASS.TWO);
}
}
log.info("scan java files done, number of symbols created: " + generatedSymbols.size());
this.cache.store(symbolsCacheKey, javaFiles, generatedSymbols, dependencyTracker.getAllDependencies(), CachedSymbol.class);
@@ -542,9 +550,8 @@ public class SpringIndexerJava implements SpringIndexer {
javaFiles.length, "Spring Tools: Indexing Java Sources for '" + project.getElementName() + "'");
try {
final boolean ignoreMethodBodies = SCAN_PASS.ONE.equals(pass);
ASTParser parser = createParser(project, ignoreMethodBodies);
List<String> nextPassFiles = new ArrayList<>();
final boolean ignoreMethodBodies = SCAN_PASS.ONE.equals(pass);
FileASTRequestor requestor = new FileASTRequestor() {
@@ -565,6 +572,7 @@ public class SpringIndexerJava implements SpringIndexer {
}
};
ASTParser parser = createParser(project, ignoreMethodBodies);
parser.createASTs(javaFiles, null, new String[0], requestor, null);
return (String[]) nextPassFiles.toArray(new String[nextPassFiles.size()]);
@@ -830,6 +838,24 @@ public class SpringIndexerJava implements SpringIndexer {
// return new IndexCacheKey(project.getElementName(), "java", elementType, DigestUtils.md5Hex(GENERATION + "-" + classpathIdentifier).toUpperCase());
return new IndexCacheKey(project.getElementName(), "java", elementType, DigestUtils.md5Hex(GENERATION + "-" + validationSeveritySettings.toString() + "-" + classpathIdentifier).toUpperCase());
}
private List<String[]> createChunks(String[] sourceArray, int chunkSize) {
int numberOfChunks = (int) Math.ceil((double) sourceArray.length / chunkSize);
List<String[]> result = new ArrayList<>();
for (int i = 0; i < numberOfChunks; i++) {
int startIdx = i * chunkSize;
int endIdx = Math.min(startIdx + chunkSize, sourceArray.length);
String[] chunk = new String[endIdx - startIdx];
System.arraycopy(sourceArray, startIdx, chunk, 0, endIdx - startIdx);
result.add(chunk);
}
return result;
}
public void setScanTestJavaSources(boolean scanTestJavaSources) {
if (this.scanTestJavaSources != scanTestJavaSources) {
@@ -900,10 +926,14 @@ public class SpringIndexerJava implements SpringIndexer {
this.fileScanListener = fileScanListener;
}
public void setScanChunkSize(int chunkSize) {
this.scanChunkSize = chunkSize;
}
private void fileScannedEvent(String file) {
if (fileScanListener != null) {
fileScanListener.fileScanned(file);
}
}
}

1
headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/java/utils/test/SpringIndexerMultipleFilesTest.java
View File

@@ -70,6 +70,7 @@ public class SpringIndexerMultipleFilesTest {
public void setup() throws Exception {
harness.intialize(null);
indexer.configureIndexer(SymbolIndexConfig.builder().scanXml(false).build());
indexer.getJavaIndexer().setScanChunkSize(2);
directory = new File(ProjectsHarness.class.getResource("/test-projects/test-annotation-indexing-parent/test-annotation-indexing/").toURI());
projectDir = directory.toURI().toString();

21
headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/java/utils/test/SpringIndexerTest.java
View File

@@ -58,6 +58,7 @@ public class SpringIndexerTest {
public void setup() throws Exception {
harness.intialize(null);
indexer.configureIndexer(SymbolIndexConfig.builder().scanXml(false).build());
indexer.getJavaIndexer().setScanChunkSize(2);
directory = new File(ProjectsHarness.class.getResource("/test-projects/test-annotation-indexing-parent/test-annotation-indexing/").toURI());
projectDir = directory.toURI().toString();
@@ -73,7 +74,7 @@ public class SpringIndexerTest {
void testScanningAllAnnotationsSimpleProjectUpfront() throws Exception {
List<? extends WorkspaceSymbol> allSymbols = indexer.getAllSymbols("");
assertEquals(7, allSymbols.size());
assertEquals(25, allSymbols.size());
String docUri = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString();
assertTrue(containsSymbol(allSymbols, "@+ 'mainClass' (@SpringBootApplication <: @SpringBootConfiguration, @Configuration, @Component) MainClass", docUri, 6, 0, 6, 22));
@@ -96,13 +97,13 @@ public class SpringIndexerTest {
indexer.configureIndexer(SymbolIndexConfig.builder().scanTestJavaSources(true).build());
List<? extends WorkspaceSymbol> allSymbols = indexer.getAllSymbols("");
assertEquals(8, allSymbols.size());
assertEquals(26, allSymbols.size());
String docUri = directory.toPath().resolve("src/test/java/demo/ApplicationTests.java").toUri().toString();
assertTrue(containsSymbol(allSymbols, "@SpringBootTest", docUri, 8, 0, 8, 15));
indexer.configureIndexer(SymbolIndexConfig.builder().scanTestJavaSources(false).build());
allSymbols = indexer.getAllSymbols("");
assertEquals(7, allSymbols.size());
assertEquals(25, allSymbols.size());
assertFalse(containsSymbol(allSymbols, "@SpringBootTest", docUri, 8, 0, 8, 15));
}
@@ -131,7 +132,7 @@ public class SpringIndexerTest {
void testScanningAllAnnotationsMultiModuleProjectUpfront() throws Exception {
List<? extends WorkspaceSymbol> allSymbols = indexer.getAllSymbols("");
assertEquals(7, allSymbols.size());
assertEquals(25, allSymbols.size());
String docUri = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString();
assertTrue(containsSymbol(allSymbols, "@+ 'mainClass' (@SpringBootApplication <: @SpringBootConfiguration, @Configuration, @Component) MainClass", docUri, 6, 0, 6, 22));
@@ -169,7 +170,7 @@ public class SpringIndexerTest {
// check for updated index in all symbols
List<? extends WorkspaceSymbol> allSymbols = indexer.getAllSymbols("");
assertEquals(7, allSymbols.size());
assertEquals(25, allSymbols.size());
String docUri = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString();
assertTrue(containsSymbol(allSymbols, "@+ 'mainClass' (@SpringBootApplication <: @SpringBootConfiguration, @Configuration, @Component) MainClass", docUri, 6, 0, 6, 22));
@@ -196,7 +197,7 @@ public class SpringIndexerTest {
assertEquals(0, symbols.size());
List<? extends WorkspaceSymbol> allSymbols = indexer.getAllSymbols("");
assertEquals(7, allSymbols.size());
assertEquals(25, allSymbols.size());
try {
// create document and update index
@@ -230,7 +231,7 @@ public class SpringIndexerTest {
// check for updated index in all symbols
allSymbols = indexer.getAllSymbols("");
assertEquals(9, allSymbols.size());
assertEquals(27, allSymbols.size());
String docUri = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString();
assertTrue(containsSymbol(allSymbols, "@+ 'mainClass' (@SpringBootApplication <: @SpringBootConfiguration, @Configuration, @Component) MainClass", docUri, 6, 0, 6, 22));
@@ -269,7 +270,7 @@ public class SpringIndexerTest {
// check for updated index in all symbols
List<? extends WorkspaceSymbol> allSymbols = indexer.getAllSymbols("");
assertEquals(5, allSymbols.size());
assertEquals(23, allSymbols.size());
String docUri = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString();
assertTrue(containsSymbol(allSymbols, "@+ 'mainClass' (@SpringBootApplication <: @SpringBootConfiguration, @Configuration, @Component) MainClass", docUri, 6, 0, 6, 22));
@@ -287,7 +288,7 @@ public class SpringIndexerTest {
void testFilterSymbolsUsingQueryString() throws Exception {
List<? extends WorkspaceSymbol> allSymbols = indexer.getAllSymbols("mapp");
assertEquals(6, allSymbols.size());
assertEquals(24, allSymbols.size());
String docUri = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString();
assertTrue(containsSymbol(allSymbols, "@/embedded-foo-mapping", docUri, 17, 1, 17, 41));
@@ -326,7 +327,7 @@ public class SpringIndexerTest {
@Test
void testDeleteProject() throws Exception {
List<? extends WorkspaceSymbol> allSymbols = indexer.getAllSymbols("");
assertEquals(7, allSymbols.size());
assertEquals(25, allSymbols.size());
CompletableFuture<Void> deleteProject = indexer.deleteProject(project);
deleteProject.get(5, TimeUnit.SECONDS);

17
headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass1.java Normal file
View File

@@ -0,0 +1,17 @@
package org.test.manyfiles;
import org.springframework.web.bind.annotation.RequestMapping;
public class SimpleMappingClass1 {
@RequestMapping("mapping1")
public String hello1() {
return "hello1";
}
@RequestMapping("mapping2")
public String hello2() {
return "hello2";
}
}

17
headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass2.java Normal file
View File

@@ -0,0 +1,17 @@
package org.test.manyfiles;
import org.springframework.web.bind.annotation.RequestMapping;
public class SimpleMappingClass2 {
@RequestMapping("mapping1")
public String hello1() {
return "hello1";
}
@RequestMapping("mapping2")
public String hello2() {
return "hello2";
}
}

17
headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass3.java Normal file
View File

@@ -0,0 +1,17 @@
package org.test.manyfiles;
import org.springframework.web.bind.annotation.RequestMapping;
public class SimpleMappingClass3 {
@RequestMapping("mapping1")
public String hello1() {
return "hello1";
}
@RequestMapping("mapping2")
public String hello2() {
return "hello2";
}
}

17
headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass4.java Normal file
View File

@@ -0,0 +1,17 @@
package org.test.manyfiles;
import org.springframework.web.bind.annotation.RequestMapping;
public class SimpleMappingClass4 {
@RequestMapping("mapping1")
public String hello1() {
return "hello1";
}
@RequestMapping("mapping2")
public String hello2() {
return "hello2";
}
}

17
headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass5.java Normal file
View File

@@ -0,0 +1,17 @@
package org.test.manyfiles;
import org.springframework.web.bind.annotation.RequestMapping;
public class SimpleMappingClass5 {
@RequestMapping("mapping1")
public String hello1() {
return "hello1";
}
@RequestMapping("mapping2")
public String hello2() {
return "hello2";
}
}

17
headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass6.java Normal file
View File

@@ -0,0 +1,17 @@
package org.test.manyfiles;
import org.springframework.web.bind.annotation.RequestMapping;
public class SimpleMappingClass6 {
@RequestMapping("mapping1")
public String hello1() {
return "hello1";
}
@RequestMapping("mapping2")
public String hello2() {
return "hello2";
}
}

17
headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass7.java Normal file
View File

@@ -0,0 +1,17 @@
package org.test.manyfiles;
import org.springframework.web.bind.annotation.RequestMapping;
public class SimpleMappingClass7 {
@RequestMapping("mapping1")
public String hello1() {
return "hello1";
}
@RequestMapping("mapping2")
public String hello2() {
return "hello2";
}
}

17
headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass8.java Normal file
View File

@@ -0,0 +1,17 @@
package org.test.manyfiles;
import org.springframework.web.bind.annotation.RequestMapping;
public class SimpleMappingClass8 {
@RequestMapping("mapping1")
public String hello1() {
return "hello1";
}
@RequestMapping("mapping2")
public String hello2() {
return "hello2";
}
}

17
headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass9.java Normal file
View File

@@ -0,0 +1,17 @@
package org.test.manyfiles;
import org.springframework.web.bind.annotation.RequestMapping;
public class SimpleMappingClass9 {
@RequestMapping("mapping1")
public String hello1() {
return "hello1";
}
@RequestMapping("mapping2")
public String hello2() {
return "hello2";
}
}