diff --git a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/utils/SpringIndexerJava.java b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/utils/SpringIndexerJava.java index 738b5cbaa..2a316cfec 100644 --- a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/utils/SpringIndexerJava.java +++ b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/utils/SpringIndexerJava.java @@ -59,8 +59,8 @@ import org.springframework.ide.vscode.boot.java.beans.CachedBean; import org.springframework.ide.vscode.boot.java.handlers.EnhancedSymbolInformation; import org.springframework.ide.vscode.boot.java.handlers.SymbolProvider; import org.springframework.ide.vscode.boot.java.reconcilers.CachedDiagnostics; -import org.springframework.ide.vscode.boot.java.reconcilers.RequiredCompleteAstException; import org.springframework.ide.vscode.boot.java.reconcilers.JdtReconciler; +import org.springframework.ide.vscode.boot.java.reconcilers.RequiredCompleteAstException; import org.springframework.ide.vscode.commons.java.IClasspath; import org.springframework.ide.vscode.commons.java.IClasspathUtil; import org.springframework.ide.vscode.commons.java.IJavaProject; @@ -108,6 +108,7 @@ public class SpringIndexerJava implements SpringIndexer { private boolean scanTestJavaSources = false; private JsonObject validationSeveritySettings; + private int scanChunkSize = 1000; private FileScanListener fileScanListener = null; //used by test code only @@ -362,7 +363,6 @@ public class SpringIndexerJava implements SpringIndexer { private Set scanFilesInternally(IJavaProject project, DocumentDescriptor[] docs) throws Exception { final boolean ignoreMethodBodies = false; - ASTParser parser = createParser(project, ignoreMethodBodies); // this is to keep track of already scanned files to avoid endless loops due to circular dependencies Set scannedTypes = new HashSet<>(); @@ -418,7 +418,11 @@ public class SpringIndexerJava implements SpringIndexer { } }; - parser.createASTs(javaFiles, null, new String[0], requestor, null); + List chunks = createChunks(javaFiles, this.scanChunkSize); + for(int i = 0; i < chunks.size(); i++) { + ASTParser parser = createParser(project, ignoreMethodBodies); + parser.createASTs(chunks.get(i), null, new String[0], requestor, null); + } EnhancedSymbolInformation[] symbols = generatedSymbols.stream().map(cachedSymbol -> cachedSymbol.getEnhancedSymbol()).toArray(EnhancedSymbolInformation[]::new); Bean[] beans = generatedBeans.stream().filter(cachedBean -> cachedBean.getBean() != null).map(cachedBean -> cachedBean.getBean()).toArray(Bean[]::new); @@ -497,14 +501,18 @@ public class SpringIndexerJava implements SpringIndexer { } }; - String[] pass2Files = scanFiles(project, javaFiles, generatedSymbols, generatedBeans, diagnosticsAggregator, SCAN_PASS.ONE); - if (pass2Files.length > 0) { + List chunks = createChunks(javaFiles, this.scanChunkSize); + for (int i = 0; i < chunks.size(); i++) { - log.info("scan java files, AST parse, pass 2 for files: {}", javaFiles.length); - - scanFiles(project, pass2Files, generatedSymbols, generatedBeans, diagnosticsAggregator, SCAN_PASS.TWO); - } + log.info("scan java files, AST parse, chunk {} for files: {}", i, javaFiles.length); + String[] pass2Files = scanFiles(project, chunks.get(i), generatedSymbols, generatedBeans, diagnosticsAggregator, SCAN_PASS.ONE); + if (pass2Files.length > 0) { + log.info("scan java files, AST parse, pass 2, chunk {} for files: {}", i, javaFiles.length); + scanFiles(project, pass2Files, generatedSymbols, generatedBeans, diagnosticsAggregator, SCAN_PASS.TWO); + } + } + log.info("scan java files done, number of symbols created: " + generatedSymbols.size()); this.cache.store(symbolsCacheKey, javaFiles, generatedSymbols, dependencyTracker.getAllDependencies(), CachedSymbol.class); @@ -542,9 +550,8 @@ public class SpringIndexerJava implements SpringIndexer { javaFiles.length, "Spring Tools: Indexing Java Sources for '" + project.getElementName() + "'"); try { - final boolean ignoreMethodBodies = SCAN_PASS.ONE.equals(pass); - ASTParser parser = createParser(project, ignoreMethodBodies); List nextPassFiles = new ArrayList<>(); + final boolean ignoreMethodBodies = SCAN_PASS.ONE.equals(pass); FileASTRequestor requestor = new FileASTRequestor() { @@ -565,6 +572,7 @@ public class SpringIndexerJava implements SpringIndexer { } }; + ASTParser parser = createParser(project, ignoreMethodBodies); parser.createASTs(javaFiles, null, new String[0], requestor, null); return (String[]) nextPassFiles.toArray(new String[nextPassFiles.size()]); @@ -830,6 +838,24 @@ public class SpringIndexerJava implements SpringIndexer { // return new IndexCacheKey(project.getElementName(), "java", elementType, DigestUtils.md5Hex(GENERATION + "-" + classpathIdentifier).toUpperCase()); return new IndexCacheKey(project.getElementName(), "java", elementType, DigestUtils.md5Hex(GENERATION + "-" + validationSeveritySettings.toString() + "-" + classpathIdentifier).toUpperCase()); } + + private List createChunks(String[] sourceArray, int chunkSize) { + + int numberOfChunks = (int) Math.ceil((double) sourceArray.length / chunkSize); + List result = new ArrayList<>(); + + for (int i = 0; i < numberOfChunks; i++) { + int startIdx = i * chunkSize; + int endIdx = Math.min(startIdx + chunkSize, sourceArray.length); + + String[] chunk = new String[endIdx - startIdx]; + System.arraycopy(sourceArray, startIdx, chunk, 0, endIdx - startIdx); + + result.add(chunk); + } + + return result; + } public void setScanTestJavaSources(boolean scanTestJavaSources) { if (this.scanTestJavaSources != scanTestJavaSources) { @@ -900,10 +926,14 @@ public class SpringIndexerJava implements SpringIndexer { this.fileScanListener = fileScanListener; } + public void setScanChunkSize(int chunkSize) { + this.scanChunkSize = chunkSize; + } + private void fileScannedEvent(String file) { if (fileScanListener != null) { fileScanListener.fileScanned(file); } } - + } diff --git a/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/java/utils/test/SpringIndexerMultipleFilesTest.java b/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/java/utils/test/SpringIndexerMultipleFilesTest.java index 8e165a556..10928d24d 100644 --- a/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/java/utils/test/SpringIndexerMultipleFilesTest.java +++ b/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/java/utils/test/SpringIndexerMultipleFilesTest.java @@ -70,6 +70,7 @@ public class SpringIndexerMultipleFilesTest { public void setup() throws Exception { harness.intialize(null); indexer.configureIndexer(SymbolIndexConfig.builder().scanXml(false).build()); + indexer.getJavaIndexer().setScanChunkSize(2); directory = new File(ProjectsHarness.class.getResource("/test-projects/test-annotation-indexing-parent/test-annotation-indexing/").toURI()); projectDir = directory.toURI().toString(); diff --git a/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/java/utils/test/SpringIndexerTest.java b/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/java/utils/test/SpringIndexerTest.java index faf9bd4ac..7682f3384 100644 --- a/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/java/utils/test/SpringIndexerTest.java +++ b/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/java/utils/test/SpringIndexerTest.java @@ -58,6 +58,7 @@ public class SpringIndexerTest { public void setup() throws Exception { harness.intialize(null); indexer.configureIndexer(SymbolIndexConfig.builder().scanXml(false).build()); + indexer.getJavaIndexer().setScanChunkSize(2); directory = new File(ProjectsHarness.class.getResource("/test-projects/test-annotation-indexing-parent/test-annotation-indexing/").toURI()); projectDir = directory.toURI().toString(); @@ -73,7 +74,7 @@ public class SpringIndexerTest { void testScanningAllAnnotationsSimpleProjectUpfront() throws Exception { List allSymbols = indexer.getAllSymbols(""); - assertEquals(7, allSymbols.size()); + assertEquals(25, allSymbols.size()); String docUri = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString(); assertTrue(containsSymbol(allSymbols, "@+ 'mainClass' (@SpringBootApplication <: @SpringBootConfiguration, @Configuration, @Component) MainClass", docUri, 6, 0, 6, 22)); @@ -96,13 +97,13 @@ public class SpringIndexerTest { indexer.configureIndexer(SymbolIndexConfig.builder().scanTestJavaSources(true).build()); List allSymbols = indexer.getAllSymbols(""); - assertEquals(8, allSymbols.size()); + assertEquals(26, allSymbols.size()); String docUri = directory.toPath().resolve("src/test/java/demo/ApplicationTests.java").toUri().toString(); assertTrue(containsSymbol(allSymbols, "@SpringBootTest", docUri, 8, 0, 8, 15)); indexer.configureIndexer(SymbolIndexConfig.builder().scanTestJavaSources(false).build()); allSymbols = indexer.getAllSymbols(""); - assertEquals(7, allSymbols.size()); + assertEquals(25, allSymbols.size()); assertFalse(containsSymbol(allSymbols, "@SpringBootTest", docUri, 8, 0, 8, 15)); } @@ -131,7 +132,7 @@ public class SpringIndexerTest { void testScanningAllAnnotationsMultiModuleProjectUpfront() throws Exception { List allSymbols = indexer.getAllSymbols(""); - assertEquals(7, allSymbols.size()); + assertEquals(25, allSymbols.size()); String docUri = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString(); assertTrue(containsSymbol(allSymbols, "@+ 'mainClass' (@SpringBootApplication <: @SpringBootConfiguration, @Configuration, @Component) MainClass", docUri, 6, 0, 6, 22)); @@ -169,7 +170,7 @@ public class SpringIndexerTest { // check for updated index in all symbols List allSymbols = indexer.getAllSymbols(""); - assertEquals(7, allSymbols.size()); + assertEquals(25, allSymbols.size()); String docUri = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString(); assertTrue(containsSymbol(allSymbols, "@+ 'mainClass' (@SpringBootApplication <: @SpringBootConfiguration, @Configuration, @Component) MainClass", docUri, 6, 0, 6, 22)); @@ -196,7 +197,7 @@ public class SpringIndexerTest { assertEquals(0, symbols.size()); List allSymbols = indexer.getAllSymbols(""); - assertEquals(7, allSymbols.size()); + assertEquals(25, allSymbols.size()); try { // create document and update index @@ -230,7 +231,7 @@ public class SpringIndexerTest { // check for updated index in all symbols allSymbols = indexer.getAllSymbols(""); - assertEquals(9, allSymbols.size()); + assertEquals(27, allSymbols.size()); String docUri = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString(); assertTrue(containsSymbol(allSymbols, "@+ 'mainClass' (@SpringBootApplication <: @SpringBootConfiguration, @Configuration, @Component) MainClass", docUri, 6, 0, 6, 22)); @@ -269,7 +270,7 @@ public class SpringIndexerTest { // check for updated index in all symbols List allSymbols = indexer.getAllSymbols(""); - assertEquals(5, allSymbols.size()); + assertEquals(23, allSymbols.size()); String docUri = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString(); assertTrue(containsSymbol(allSymbols, "@+ 'mainClass' (@SpringBootApplication <: @SpringBootConfiguration, @Configuration, @Component) MainClass", docUri, 6, 0, 6, 22)); @@ -287,7 +288,7 @@ public class SpringIndexerTest { void testFilterSymbolsUsingQueryString() throws Exception { List allSymbols = indexer.getAllSymbols("mapp"); - assertEquals(6, allSymbols.size()); + assertEquals(24, allSymbols.size()); String docUri = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString(); assertTrue(containsSymbol(allSymbols, "@/embedded-foo-mapping", docUri, 17, 1, 17, 41)); @@ -326,7 +327,7 @@ public class SpringIndexerTest { @Test void testDeleteProject() throws Exception { List allSymbols = indexer.getAllSymbols(""); - assertEquals(7, allSymbols.size()); + assertEquals(25, allSymbols.size()); CompletableFuture deleteProject = indexer.deleteProject(project); deleteProject.get(5, TimeUnit.SECONDS); diff --git a/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass1.java b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass1.java new file mode 100644 index 000000000..e16a7dec7 --- /dev/null +++ b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass1.java @@ -0,0 +1,17 @@ +package org.test.manyfiles; + +import org.springframework.web.bind.annotation.RequestMapping; + +public class SimpleMappingClass1 { + + @RequestMapping("mapping1") + public String hello1() { + return "hello1"; + } + + @RequestMapping("mapping2") + public String hello2() { + return "hello2"; + } + +} diff --git a/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass2.java b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass2.java new file mode 100644 index 000000000..5841303a0 --- /dev/null +++ b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass2.java @@ -0,0 +1,17 @@ +package org.test.manyfiles; + +import org.springframework.web.bind.annotation.RequestMapping; + +public class SimpleMappingClass2 { + + @RequestMapping("mapping1") + public String hello1() { + return "hello1"; + } + + @RequestMapping("mapping2") + public String hello2() { + return "hello2"; + } + +} diff --git a/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass3.java b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass3.java new file mode 100644 index 000000000..81c665e1b --- /dev/null +++ b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass3.java @@ -0,0 +1,17 @@ +package org.test.manyfiles; + +import org.springframework.web.bind.annotation.RequestMapping; + +public class SimpleMappingClass3 { + + @RequestMapping("mapping1") + public String hello1() { + return "hello1"; + } + + @RequestMapping("mapping2") + public String hello2() { + return "hello2"; + } + +} diff --git a/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass4.java b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass4.java new file mode 100644 index 000000000..f2a5be3ab --- /dev/null +++ b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass4.java @@ -0,0 +1,17 @@ +package org.test.manyfiles; + +import org.springframework.web.bind.annotation.RequestMapping; + +public class SimpleMappingClass4 { + + @RequestMapping("mapping1") + public String hello1() { + return "hello1"; + } + + @RequestMapping("mapping2") + public String hello2() { + return "hello2"; + } + +} diff --git a/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass5.java b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass5.java new file mode 100644 index 000000000..676f9a7b4 --- /dev/null +++ b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass5.java @@ -0,0 +1,17 @@ +package org.test.manyfiles; + +import org.springframework.web.bind.annotation.RequestMapping; + +public class SimpleMappingClass5 { + + @RequestMapping("mapping1") + public String hello1() { + return "hello1"; + } + + @RequestMapping("mapping2") + public String hello2() { + return "hello2"; + } + +} diff --git a/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass6.java b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass6.java new file mode 100644 index 000000000..bf4c81a81 --- /dev/null +++ b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass6.java @@ -0,0 +1,17 @@ +package org.test.manyfiles; + +import org.springframework.web.bind.annotation.RequestMapping; + +public class SimpleMappingClass6 { + + @RequestMapping("mapping1") + public String hello1() { + return "hello1"; + } + + @RequestMapping("mapping2") + public String hello2() { + return "hello2"; + } + +} diff --git a/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass7.java b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass7.java new file mode 100644 index 000000000..b275b7ea1 --- /dev/null +++ b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass7.java @@ -0,0 +1,17 @@ +package org.test.manyfiles; + +import org.springframework.web.bind.annotation.RequestMapping; + +public class SimpleMappingClass7 { + + @RequestMapping("mapping1") + public String hello1() { + return "hello1"; + } + + @RequestMapping("mapping2") + public String hello2() { + return "hello2"; + } + +} diff --git a/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass8.java b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass8.java new file mode 100644 index 000000000..cf767b0df --- /dev/null +++ b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass8.java @@ -0,0 +1,17 @@ +package org.test.manyfiles; + +import org.springframework.web.bind.annotation.RequestMapping; + +public class SimpleMappingClass8 { + + @RequestMapping("mapping1") + public String hello1() { + return "hello1"; + } + + @RequestMapping("mapping2") + public String hello2() { + return "hello2"; + } + +} diff --git a/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass9.java b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass9.java new file mode 100644 index 000000000..abb5e1651 --- /dev/null +++ b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/manyfiles/SimpleMappingClass9.java @@ -0,0 +1,17 @@ +package org.test.manyfiles; + +import org.springframework.web.bind.annotation.RequestMapping; + +public class SimpleMappingClass9 { + + @RequestMapping("mapping1") + public String hello1() { + return "hello1"; + } + + @RequestMapping("mapping2") + public String hello2() { + return "hello2"; + } + +}