diff --git a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/index/cache/IndexCacheOnDisc.java b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/index/cache/IndexCacheOnDisc.java
index dd0e296e5..ccf52520e 100644
--- a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/index/cache/IndexCacheOnDisc.java
+++ b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/index/cache/IndexCacheOnDisc.java
@@ -308,6 +308,7 @@ public class IndexCacheOnDisc implements IndexCache {
 					&& key.getIndexer().equals(cacheKey.getIndexer())
 					&& key.getCategory().equals(cacheKey.getCategory())) {
 				cacheFiles[i].delete();
+				this.stores.remove(key);
 			}
 			// cleanup old cache files without category information (pre 4.19.1 release)
 			else if (key != null && !key.equals(cacheKey)
diff --git a/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/index/cache/test/IndexCacheOnDiscTest.java b/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/index/cache/test/IndexCacheOnDiscTest.java
index aa4a99177..4a2c7d5a0 100644
--- a/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/index/cache/test/IndexCacheOnDiscTest.java
+++ b/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/index/cache/test/IndexCacheOnDiscTest.java
@@ -217,6 +217,20 @@ public class IndexCacheOnDiscTest {
         assertFalse(Files.exists(tempDir.resolve(Paths.get(key1.toString() + ".json"))));
     }
 
+    @Test
+    void testDoNotRetrieveOldCacheDataIfNewerVersionIsStored() throws Exception {
+        IndexCacheKey key1 = CACHE_KEY_VERSION_1;
+        IndexCacheKey key2 = new IndexCacheKey("someProject", "someIndexer", "someCategory", "2");
+
+        cache.store(key1, new String[0], new ArrayList<>(), null, CachedSymbol.class);
+        assertNotNull(cache.retrieve(key1, new String[0], CachedSymbol.class));
+        assertNull(cache.retrieve(key2, new String[0], CachedSymbol.class));
+
+        cache.store(key2, new String[0], new ArrayList<>(), null, CachedSymbol.class);
+        assertNull(cache.retrieve(key1, new String[0], CachedSymbol.class));
+        assertNotNull(cache.retrieve(key2, new String[0], CachedSymbol.class));
+    }
+
     @Test
     void testDeleteOldCacheFileFromPreviousReleasesIfNewOneIsStored() throws Exception {
         IndexCacheKey key1 = new IndexCacheKey("someProject", "someIndexer", "", "2");