From 96ce86171b94f67fb8c3842cfc4bdc18b4d80f17 Mon Sep 17 00:00:00 2001 From: Martin Lippert Date: Mon, 22 Apr 2024 09:55:25 +0200 Subject: [PATCH] GH-1231: symbol indexer now reacts to folder changes accordingly, even if there are no events for contained files being send immediately --- .../vscode/boot/app/SpringSymbolIndex.java | 169 ++++++++++-------- .../java/utils/test/SpringIndexerTest.java | 99 +++++++++- .../subdir3/subdir4/subfile-level4.txt | 0 .../subdir3/subdir4/subfile2-level4.txt | 0 .../test/subdir/subdir2/subfile-level2.txt | 0 .../java/org/test/subdir/subfile-level1.txt | 0 6 files changed, 191 insertions(+), 77 deletions(-) create mode 100644 headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/subdir/subdir2/subdir3/subdir4/subfile-level4.txt create mode 100644 headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/subdir/subdir2/subdir3/subdir4/subfile2-level4.txt create mode 100644 headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/subdir/subdir2/subfile-level2.txt create mode 100644 headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/subdir/subfile-level1.txt diff --git a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/app/SpringSymbolIndex.java b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/app/SpringSymbolIndex.java index acf13955d..1f5ba3375 100644 --- a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/app/SpringSymbolIndex.java +++ b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/app/SpringSymbolIndex.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2017, 2023 Pivotal, Inc. + * Copyright (c) 2017, 2024 Pivotal, Inc. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -11,13 +11,17 @@ package org.springframework.ide.vscode.boot.app; import java.io.File; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -144,8 +148,6 @@ public class SpringSymbolIndex implements InitializingBean, SpringIndex { private SpringIndexerJava springIndexerJava; private SpringFactoriesIndexer factoriesIndexer; - private String watchXMLDeleteRegistration; - private String watchXMLCreatedRegistration; private String watchXMLChangedRegistration; // Futures resolved when project is initialized/indexed @@ -294,15 +296,19 @@ public class SpringSymbolIndex implements InitializingBean, SpringIndex { List globPattern = Stream.concat(Arrays.stream(springIndexerJava.getFileWatchPatterns()), Arrays.stream(factoriesIndexer.getFileWatchPatterns())) .collect(Collectors.toList()); - getWorkspaceService().getFileObserver().onFilesDeleted(globPattern, (files) -> { - deleteDocuments(files); - }); - getWorkspaceService().getFileObserver().onFilesCreated(globPattern, (files) -> { - createDocuments(files); - }); getWorkspaceService().getFileObserver().onFilesChanged(globPattern, (files) -> { updateDocuments(files, "file changed"); }); + + // watch for creation of files and folders (basically everything) to catch folder rename events as well + getWorkspaceService().getFileObserver().onFilesCreated(List.of("**/*"), (files) -> { + createDocuments(files); + }); + + // watch for deletion of files and folders (basically everything) to catch folder rename events as well + getWorkspaceService().getFileObserver().onFilesDeleted(List.of("**/*"), (files) -> { + deleteDocuments(files); + }); } public void configurationChanged(SymbolIndexConfig config) { @@ -337,14 +343,6 @@ public class SpringSymbolIndex implements InitializingBean, SpringIndex { private void addXmlFileListeners(List globPattern) { removeXmlFileListeners(); - watchXMLDeleteRegistration = getWorkspaceService().getFileObserver().onFilesDeleted(globPattern, - (files) -> { - deleteDocuments(files); - }); - watchXMLCreatedRegistration = getWorkspaceService().getFileObserver().onFilesCreated(globPattern, - (files) -> { - createDocuments(files); - }); watchXMLChangedRegistration = getWorkspaceService().getFileObserver().onFilesChanged(globPattern, (files) -> { updateDocuments(files, "xml changed"); @@ -356,14 +354,6 @@ public class SpringSymbolIndex implements InitializingBean, SpringIndex { getWorkspaceService().getFileObserver().unsubscribe(watchXMLChangedRegistration); watchXMLChangedRegistration = null; } - if (watchXMLCreatedRegistration != null) { - getWorkspaceService().getFileObserver().unsubscribe(watchXMLCreatedRegistration); - watchXMLCreatedRegistration = null; - } - if (watchXMLDeleteRegistration != null) { - getWorkspaceService().getFileObserver().unsubscribe(watchXMLDeleteRegistration); - watchXMLDeleteRegistration = null; - } } public void shutdown() { @@ -455,35 +445,15 @@ public class SpringSymbolIndex implements InitializingBean, SpringIndex { } public CompletableFuture createDocument(String docURI) { - synchronized(this) { - List> futures = new ArrayList<>(); - - for (SpringIndexer indexer : this.indexers) { - if (indexer.isInterestedIn(docURI)) { - Optional maybeProject = projectFinder().find(new TextDocumentIdentifier(docURI)); - - if (maybeProject.isPresent()) { - try { - DocumentDescriptor newDoc = createUpdatedDoc(docURI); - futures.add(updateItems(maybeProject.get(), new DocumentDescriptor[] {newDoc}, indexer)); - } - catch (Exception e) { - log.error("", e); - futures.add(Futures.error(e)); - } - } - } - } - - CompletableFuture future = CompletableFuture.allOf((CompletableFuture[]) futures.toArray(new CompletableFuture[futures.size()])); - future = future.thenAccept(v -> server.getClient().indexUpdated()).thenAccept(v -> listeners.fire(v)); - return future; - } + String[] docURIs = unfold(docURI); + return createDocuments(docURIs); } public CompletableFuture createDocuments(String[] docURIs) { synchronized(this) { List> futures = new ArrayList<>(); + + docURIs = unfold(docURIs); for (SpringIndexer indexer : this.indexers) { String[] interestingDocs = getDocumentsInterestingForIndexer(indexer, docURIs); @@ -509,6 +479,32 @@ public class SpringSymbolIndex implements InitializingBean, SpringIndex { } } + public static String[] unfold(String... docURIs) { + Set result = new HashSet<>(); + + for (int i = 0; i < docURIs.length; i++) { + File file = UriUtil.toFile(docURIs[i]); + Path path = file.toPath(); + + if (Files.isRegularFile(path)) { + result.add(docURIs[i]); + } + else if (Files.isDirectory(path)) { + try { + Files.walk(path) + .filter(Files::isRegularFile) + .map(filePath -> filePath.toAbsolutePath().toString()) + .map(filePath -> UriUtil.toUri(new File(filePath)).toASCIIString()) + .forEach(docURI -> result.add(docURI)); + } catch (IOException e) { + log.error("error unfolding: " + path.toString(), e); + } + } + } + + return (String[]) result.toArray(new String[result.size()]); + } + private JavaProjectFinder projectFinder() { return params.projectFinder; } @@ -582,14 +578,53 @@ public class SpringSymbolIndex implements InitializingBean, SpringIndex { return result; } - private String[] getDocumentsInterestingForIndexer(SpringIndexer indexer, String[] docURIs) { - return Arrays.stream(docURIs).filter(docURI -> indexer.isInterestedIn(docURI)).toArray(String[]::new); - } - private Map> getProjectMapping(Map docsToProject) { return docsToProject.keySet().stream().collect(Collectors.groupingBy(docURI -> docsToProject.get(docURI))); } + private Map> getDocsPerProjectFromPaths(String[] paths) { + Map> result = new HashMap<>(); + + for (String path : paths) { + Optional project = projectFinder().find(new TextDocumentIdentifier(path)); + if (project.isPresent()) { + result.putIfAbsent(project.get(), new HashSet<>()); + + Set docs = result.get(project.get()); + docs.addAll(getDocsFromPath(project.get(), path)); + } + } + + return result; + } + + private Collection getDocsFromPath(IJavaProject project, String path) { + List allProjectSymbols = this.symbolsByProject.get(project.getElementName()); + Set result = new HashSet<>(); + + for (EnhancedSymbolInformation symbol : allProjectSymbols) { + Either location = symbol.getSymbol().getLocation(); + + String docURI = null; + if (location.isLeft()) { + docURI = location.getLeft().getUri(); + } + else if (location.isRight()) { + docURI = location.getRight().getUri(); + } + + if (docURI != null && docURI.startsWith(path)) { + result.add(docURI); + } + } + + return result; + } + + private String[] getDocumentsInterestingForIndexer(SpringIndexer indexer, String[] docURIs) { + return Arrays.stream(docURIs).filter(docURI -> indexer.isInterestedIn(docURI)).toArray(String[]::new); + } + private DocumentDescriptor createUpdatedDoc(String docURI) throws RuntimeException { try { File file = new File(new URI(docURI)); @@ -601,36 +636,18 @@ public class SpringSymbolIndex implements InitializingBean, SpringIndex { } public CompletableFuture deleteDocument(String deletedDocURI) { - synchronized(this) { - try { - Optional maybeProject = projectFinder().find(new TextDocumentIdentifier(deletedDocURI)); - if (maybeProject.isPresent()) { - DeleteItems deleteItem = new DeleteItems(maybeProject.get(), new String[] {deletedDocURI}, this.indexers); - CompletableFuture future = CompletableFuture.runAsync(deleteItem, this.updateQueue); - - this.latestScheduledTaskByProject.put(maybeProject.get().getElementName(), future); - return future; - } - } - catch (Exception e) { - log.error("", e); - return Futures.error(e); - } - } - - return null; + return deleteDocuments(new String[] {deletedDocURI}); } - public CompletableFuture deleteDocuments(String[] deletedDocURIs) { + public CompletableFuture deleteDocuments(String[] deletedPathURIs) { synchronized(this) { try { List> futures = new ArrayList<>(); - - Map projectsForDocs = getProjectsForDocs(deletedDocURIs); - Map> projectMapping = getProjectMapping(projectsForDocs); + + Map> projectMapping = getDocsPerProjectFromPaths(deletedPathURIs); for (IJavaProject project : projectMapping.keySet()) { - List docURIs = projectMapping.get(project); + Set docURIs = projectMapping.get(project); DeleteItems deleteItems = new DeleteItems(project, (String[]) docURIs.toArray(new String[docURIs.size()]), this.indexers); CompletableFuture future = CompletableFuture.runAsync(deleteItems, this.updateQueue); diff --git a/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/java/utils/test/SpringIndexerTest.java b/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/java/utils/test/SpringIndexerTest.java index 7682f3384..b1b359024 100644 --- a/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/java/utils/test/SpringIndexerTest.java +++ b/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/java/utils/test/SpringIndexerTest.java @@ -16,6 +16,7 @@ import java.io.File; import java.net.URI; import java.util.Iterator; import java.util.List; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -263,7 +264,7 @@ public class SpringIndexerTest { assertFalse(indexer.getSymbols(deletedDocURI).isEmpty()); //We have symbols before deletion? CompletableFuture deleteFuture = indexer.deleteDocument(deletedDocURI); - deleteFuture.get(5, TimeUnit.HOURS); + deleteFuture.get(5, TimeUnit.MINUTES); // check for updated index per document Assert.noElements(indexer.getSymbols(deletedDocURI)); @@ -284,6 +285,78 @@ public class SpringIndexerTest { assertTrue(containsSymbol(allSymbols, "@Configurable", docUri, 4, 0, 4, 13)); } + @Test + void testRemoveSymbolsFromDeletedFolder() throws Exception { + // update document and update index + String deletedFolderURI = directory.toPath().resolve("src/main/java/org/test/sub").toUri().toString(); + String fileInFolderDocURI = directory.toPath().resolve("src/main/java/org/test/sub/MappingClassSubpackage.java").toUri().toString(); + + // indexer has symbols for doc in folder + List symbolsInDoc = indexer.getSymbols(fileInFolderDocURI); + assertEquals(1, symbolsInDoc.size()); + + int symbolCountBeforeDelete = indexer.getAllSymbols("").size(); + + // delete folder + CompletableFuture deleteFuture = indexer.deleteDocuments(new String[] {deletedFolderURI}); + deleteFuture.get(5, TimeUnit.MINUTES); + + // doc symbols are not around anymore + List symbolsInDocAfterDelete = indexer.getSymbols(fileInFolderDocURI); + assertEquals(0, symbolsInDocAfterDelete.size()); + + // check for updated index in all symbols + List allSymbols = indexer.getAllSymbols(""); + assertEquals(symbolCountBeforeDelete - symbolsInDoc.size(), allSymbols.size()); + + String docUri = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString(); + assertTrue(containsSymbol(allSymbols, "@+ 'mainClass' (@SpringBootApplication <: @SpringBootConfiguration, @Configuration, @Component) MainClass", docUri, 6, 0, 6, 22)); + assertTrue(containsSymbol(allSymbols, "@/embedded-foo-mapping", docUri, 17, 1, 17, 41)); + assertTrue(containsSymbol(allSymbols, "@/foo-root-mapping/embedded-foo-mapping-with-root", docUri, 27, 1, 27, 51)); + + docUri = directory.toPath().resolve("src/main/java/org/test/sub/MappingClassSubpackage.java").toUri().toString(); + assertFalse(containsSymbol(allSymbols, "@/classlevel/mapping-subpackage", docUri, 7, 1, 7, 38)); + + docUri = directory.toPath().resolve("src/main/java/org/test/ClassWithDefaultSymbol.java").toUri().toString(); + assertTrue(containsSymbol(allSymbols, "@Configurable", docUri, 4, 0, 4, 13)); + } + + @Test + void testAddSymbolsFromCreatedFolder() throws Exception { + // update document and update index + String deletedFolderURI = directory.toPath().resolve("src/main/java/org/test/sub").toUri().toString(); + String fileInFolderDocURI = directory.toPath().resolve("src/main/java/org/test/sub/MappingClassSubpackage.java").toUri().toString(); + + int symbolCountBeforeDelete = indexer.getAllSymbols("").size(); + + // delete folder first + CompletableFuture deleteFuture = indexer.deleteDocuments(new String[] {deletedFolderURI}); + deleteFuture.get(5, TimeUnit.SECONDS); + + // add folder back (if it was created) + CompletableFuture createdFuture = indexer.createDocuments(new String[] {deletedFolderURI}); + createdFuture.get(5, TimeUnit.SECONDS); + + // check for symbol from doc in created folder + List symbolsInDoc = indexer.getSymbols(fileInFolderDocURI); + assertEquals(1, symbolsInDoc.size()); + + // check for updated index in all symbols + List allSymbols = indexer.getAllSymbols(""); + assertEquals(symbolCountBeforeDelete, allSymbols.size()); + + String docUri = directory.toPath().resolve("src/main/java/org/test/MainClass.java").toUri().toString(); + assertTrue(containsSymbol(allSymbols, "@+ 'mainClass' (@SpringBootApplication <: @SpringBootConfiguration, @Configuration, @Component) MainClass", docUri, 6, 0, 6, 22)); + assertTrue(containsSymbol(allSymbols, "@/embedded-foo-mapping", docUri, 17, 1, 17, 41)); + assertTrue(containsSymbol(allSymbols, "@/foo-root-mapping/embedded-foo-mapping-with-root", docUri, 27, 1, 27, 51)); + + docUri = directory.toPath().resolve("src/main/java/org/test/sub/MappingClassSubpackage.java").toUri().toString(); + assertTrue(containsSymbol(allSymbols, "@/classlevel/mapping-subpackage", docUri, 7, 1, 7, 38)); + + docUri = directory.toPath().resolve("src/main/java/org/test/ClassWithDefaultSymbol.java").toUri().toString(); + assertTrue(containsSymbol(allSymbols, "@Configurable", docUri, 4, 0, 4, 13)); + } + @Test void testFilterSymbolsUsingQueryString() throws Exception { List allSymbols = indexer.getAllSymbols("mapp"); @@ -335,6 +408,30 @@ public class SpringIndexerTest { allSymbols = indexer.getAllSymbols(""); assertEquals(0, allSymbols.size()); } + + @Test + void testSimpleUnfoldUris() throws Exception { + String docUri = directory.toPath().resolve("src/main/java/org/test/sub").toUri().toString(); + + String[] docs = SpringSymbolIndex.unfold(docUri); + assertEquals(1, docs.length); + + String docToFindUri = directory.toPath().resolve("src/main/java/org/test/sub/MappingClassSubpackage.java").toUri().toString(); + assertEquals(docs[0], docToFindUri); + } + + @Test + void testDeepUnfoldUris() throws Exception { + String docUri = directory.toPath().resolve("src/main/java/org/test/subdir").toUri().toString(); + + Set docs = Set.of(SpringSymbolIndex.unfold(docUri)); + assertEquals(4, docs.size()); + + assertTrue(docs.contains(directory.toPath().resolve("src/main/java/org/test/subdir/subfile-level1.txt").toUri().toString())); + assertTrue(docs.contains(directory.toPath().resolve("src/main/java/org/test/subdir/subdir2/subfile-level2.txt").toUri().toString())); + assertTrue(docs.contains(directory.toPath().resolve("src/main/java/org/test/subdir/subdir2/subdir3/subdir4/subfile-level4.txt").toUri().toString())); + assertTrue(docs.contains(directory.toPath().resolve("src/main/java/org/test/subdir/subdir2/subdir3/subdir4/subfile2-level4.txt").toUri().toString())); + } static boolean containsSymbol(List symbols, String name, String uri) { for (Iterator iterator = symbols.iterator(); iterator.hasNext();) { diff --git a/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/subdir/subdir2/subdir3/subdir4/subfile-level4.txt b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/subdir/subdir2/subdir3/subdir4/subfile-level4.txt new file mode 100644 index 000000000..e69de29bb diff --git a/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/subdir/subdir2/subdir3/subdir4/subfile2-level4.txt b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/subdir/subdir2/subdir3/subdir4/subfile2-level4.txt new file mode 100644 index 000000000..e69de29bb diff --git a/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/subdir/subdir2/subfile-level2.txt b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/subdir/subdir2/subfile-level2.txt new file mode 100644 index 000000000..e69de29bb diff --git a/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/subdir/subfile-level1.txt b/headless-services/spring-boot-language-server/src/test/resources/test-projects/test-annotation-indexing-parent/test-annotation-indexing/src/main/java/org/test/subdir/subfile-level1.txt new file mode 100644 index 000000000..e69de29bb