Fine grain locking to avoid deadlocks in CU cache

Signed-off-by: aboyko <alex.boyko@broadcom.com>

headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/utils/CompilationUnitCache.java

@@ -216,8 +216,6 @@ public final class CompilationUnitCache implements DocumentContentProvider {
 		logger.info("CU Cache: work item submitted for doc {}", uri.toASCIIString());
 
 		if (project != null) {
-			ReadLock lock = environmentCacheLock.readLock();
-			lock.lock();
 			try {
 				CompilationUnit cu = null;
 				try {
@@ -234,10 +232,16 @@ public final class CompilationUnitCache implements DocumentContentProvider {
 				}
 	
 				if (cu != null) {
-						projectToDocs.get(project.getLocationUri(), () -> new HashSet<>()).add(uri);
+					ReadLock lock = environmentCacheLock.readLock();
-	
+					lock.lock();
-						logger.debug("CU Cache: start work on AST for {}", uri.toString());
+					try {
+						logger.info("CU Cache: start work on AST for {}", uri.toString());
 						return requestor.apply(cu);
+					} finally {
+						logger.info("CU Cache: end work on AST for {}", uri.toString());
+						lock.unlock();
+					}
+
 				}
 			}
 			catch (CancellationException e) {
@@ -246,10 +250,6 @@ public final class CompilationUnitCache implements DocumentContentProvider {
 			catch (Exception e) {
 				logger.error("", e);
 			}
-			finally {
-				logger.debug("CU Cache: end work on AST for {}", uri.toString());
-				lock.unlock();
-			}
 		}
 
 		return requestor.apply(null);
@@ -259,34 +259,49 @@ public final class CompilationUnitCache implements DocumentContentProvider {
 		CompletableFuture<CompilationUnit> cuFuture = uriToCu.getIfPresent(uri);
 		if (cuFuture == null) {
 			cuFuture = CompletableFuture.supplyAsync(() -> {
+				ReadLock lock = environmentCacheLock.readLock();
+				lock.lock();
+				logger.info("Started parsing CU for " + uri);
 				
 				try {
-					logger.info("Started parsing CU for " + uri);
 					Tuple2<List<Classpath>, INameEnvironmentWithProgress> lookupEnvTuple = loadLookupEnvTuple(project);
 					String uriStr = uri.toASCIIString();
 					String unitName = uriStr.substring(uriStr.lastIndexOf("/") + 1); // skip over '/'
 					CompilationUnit cUnit = parse2(fetchContent(uri).toCharArray(), uriStr, unitName, lookupEnvTuple.getT1(), lookupEnvTuple.getT2(),
 							annotationHierarchies.get(project.getLocationUri(), AnnotationHierarchies::new));
-
 					logger.debug("CU Cache: created new AST for {}", uri.toASCIIString());
-
 					logger.info("Parsed successfully CU for " + uri);
 					return cUnit;
 				} catch (Throwable t) {
 					// Complete future exceptionally
 					throw new CompletionException(t);
+				} finally {
+					logger.info("Finished parsing CU for {}", uri);
+					lock.unlock();
 				}
 			}, createCuExecutorThreadPool);
 			// Cache the future
 			uriToCu.put(uri, cuFuture);
 			// If CU future completed exceptionally invalidate the cache entry
-			cuFuture.exceptionally(t -> {
+			cuFuture
-				if (!(t instanceof CancellationException)) {
+				.thenAccept(cu -> {
-					logger.error("", t);
+					synchronized(CompilationUnitCache.this) {
-				}
+						try {
-				uriToCu.invalidate(uri);
+							projectToDocs.get(project.getLocationUri(), () -> new HashSet<>()).add(uri);
-				return null;
+						} catch (ExecutionException e) {
-			});
+							// shouldn't happen
+						}
+					}
+				})
+				.exceptionally(t -> {
+					if (!(t instanceof CancellationException)) {
+						logger.error("", t);
+					}
+					synchronized(CompilationUnitCache.this) {
+						uriToCu.invalidate(uri);
+					}
+					return null;
+				});
 		}
 		return cuFuture;
 	}

headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/jdt/ls/JdtLsProjectCache.java

@@ -338,43 +338,52 @@ public class JdtLsProjectCache implements InitializableJavaProjectsService, Serv
 	
 	private class JstLsClasspathListener implements ClasspathListener {
 		
+		/*
+		 * Synchronize to make non-reentrant such that events handled in predictable order 
+		 */
 		@Override
-		public void changed(Event event) {
+		public synchronized void changed(Event event) {
 			log.debug("claspath event received {}", event);
 			server.doOnInitialized(() -> {
 				try {
-					synchronized (table) {
+					String uri = UriUtil.normalize(event.projectUri);
-						String uri = UriUtil.normalize(event.projectUri);
+					log.debug("uri = {}", uri);
-						log.debug("uri = {}", uri);
+					if (event.deleted) {
-						if (event.deleted) {
+						log.debug("event.deleted = true");
-							log.debug("event.deleted = true");
+						IJavaProject deleted;
-							IJavaProject deleted = table.remove(uri);
+						synchronized (table) {
-							if (deleted!=null) {
+							deleted = table.remove(uri);
-								log.debug("removed from table = true");
+						}
-								notifyDelete(deleted);
+						// Notify outside of the lock 
-							} else {
+						if (deleted!=null) {
-								log.warn("Deleted project not removed because uri {} not found in {}", uri, table.keySet());
+							log.debug("removed from table = true");
-							}
+							notifyDelete(deleted);
 						} else {
-							log.debug("deleted = false");
+							log.warn("Deleted project not removed because uri {} not found in {}", uri, table.keySet());
-							URI projectUri = new URI(uri);
+						}
-							ClasspathData classpath = new ClasspathData(event.name, event.classpath.getEntries(), event.classpath.getJavaVersion());
+					} else {
-							IJavaProject oldProject = table.get(uri);
+						log.debug("deleted = false");
+						URI projectUri = new URI(uri);
+						ClasspathData classpath = new ClasspathData(event.name, event.classpath.getEntries(), event.classpath.getJavaVersion());
+						IJavaProject oldProject, newProject;
+						synchronized(table) {
+							oldProject = table.get(uri);
 							if (oldProject != null && classpath.equals(oldProject.getClasspath())) {
 								// nothing has changed
 								return;
 							}
-							IProjectBuild projectBuild = from(event.projectBuild);							
+							IProjectBuild projectBuild = from(event.projectBuild);
-							IJavaProject newProject = IS_JANDEX_INDEX
+							newProject = IS_JANDEX_INDEX
 									? new JavaProject(getFileObserver(), projectUri, classpath,
 											JdtLsProjectCache.this, projectBuild)
 									: new JdtLsJavaProject(server.getClient(), projectUri, classpath, JdtLsProjectCache.this, projectBuild);
 							table.put(uri, newProject);
-							if (oldProject != null) {
+						}
-								notifyChanged(newProject);
+						// Notify outside of the lock 
-							} else {
+						if (oldProject != null) {
-								notifyCreated(newProject);
+							notifyChanged(newProject);
-							}
+						} else {
+							notifyCreated(newProject);
 						}
 					}
 				} catch (Exception e) {