From e7d9d6da447752c10d9b080bab8b54939e2075d1 Mon Sep 17 00:00:00 2001 From: Martin Lippert Date: Wed, 11 Dec 2024 13:02:16 +0100 Subject: [PATCH] delta-based index cache: store md5 hash in timestamp map to identify files instead of full file path --- .../cache/IndexCacheOnDiscDeltaBased.java | 174 ++++++++++++------ 1 file changed, 117 insertions(+), 57 deletions(-) diff --git a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/index/cache/IndexCacheOnDiscDeltaBased.java b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/index/cache/IndexCacheOnDiscDeltaBased.java index 5b77c1867..dc73f46cf 100644 --- a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/index/cache/IndexCacheOnDiscDeltaBased.java +++ b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/index/cache/IndexCacheOnDiscDeltaBased.java @@ -30,8 +30,10 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.tuple.Pair; import org.eclipse.lsp4j.Location; import org.slf4j.Logger; @@ -66,7 +68,7 @@ import com.google.gson.stream.JsonReader; public class IndexCacheOnDiscDeltaBased implements IndexCache { private final File cacheDirectory; - private final Map> timestamps; + private final Map> timestamps; private static final Logger log = LoggerFactory.getLogger(IndexCacheOnDiscDeltaBased.class); @@ -106,7 +108,8 @@ public class IndexCacheOnDiscDeltaBased implements IndexCache { persist(cacheKey, new DeltaSnapshot(store), false); // update local timestamp cache - ConcurrentHashMap timestampMap = new ConcurrentHashMap<>(timestampedFiles); + ConcurrentMap timestampMap = timestampedFiles.entrySet().stream() + .collect(Collectors.toConcurrentMap(e -> InternalFileIdentifier.fromPath(e.getKey()), e -> e.getValue())); this.timestamps.put(cacheKey, timestampMap); } @@ -143,7 +146,9 @@ public class IndexCacheOnDiscDeltaBased implements IndexCache { } // update local timestamp cache - this.timestamps.put(cacheKey, new ConcurrentHashMap<>(timestampedFiles)); + ConcurrentMap timestampMap = timestampedFiles.entrySet().stream() + .collect(Collectors.toConcurrentMap(e -> InternalFileIdentifier.fromPath(e.getKey()), e -> e.getValue())); + this.timestamps.put(cacheKey, timestampMap); return Pair.of( (T[]) symbols.toArray((T[]) Array.newInstance(type, symbols.size())), @@ -168,10 +173,10 @@ public class IndexCacheOnDiscDeltaBased implements IndexCache { persist(cacheKey, new DeltaDelete(files), true); // update local timestamp cache - Map timestampsMap = this.timestamps.get(cacheKey); + Map timestampsMap = this.timestamps.get(cacheKey); if (timestampsMap != null) { for (String file : files) { - timestampsMap.remove(file); + timestampsMap.remove(InternalFileIdentifier.fromPath(file)); } } } @@ -205,8 +210,8 @@ public class IndexCacheOnDiscDeltaBased implements IndexCache { persist(cacheKey, new DeltaUpdate(deltaStore), true); // update local timestamp cache - Map timestampsMap = this.timestamps.computeIfAbsent(cacheKey, (s) -> new ConcurrentHashMap<>()); - timestampsMap.put(file, lastModified); + Map timestampsMap = this.timestamps.computeIfAbsent(cacheKey, (s) -> new ConcurrentHashMap<>()); + timestampsMap.put(InternalFileIdentifier.fromPath(file), lastModified); } @Override @@ -229,17 +234,19 @@ public class IndexCacheOnDiscDeltaBased implements IndexCache { persist(cacheKey, new DeltaUpdate(deltaStore), true); // update local timestamp cache - Map timestampsMap = this.timestamps.computeIfAbsent(cacheKey, (s) -> new ConcurrentHashMap<>()); + Map timestampsMap = this.timestamps.computeIfAbsent(cacheKey, (s) -> new ConcurrentHashMap<>()); for (int i = 0; i < files.length; i++) { - timestampsMap.put(files[i], lastModified[i]); + timestampsMap.put(InternalFileIdentifier.fromPath(files[i]), lastModified[i]); } } @Override public long getModificationTimestamp(IndexCacheKey cacheKey, String file) { - Map timestampsMap = this.timestamps.get(cacheKey); + InternalFileIdentifier fileID = InternalFileIdentifier.fromPath(file); + + Map timestampsMap = this.timestamps.get(cacheKey); if (timestampsMap != null) { - Long result = timestampsMap.get(file); + Long result = timestampsMap.get(fileID); if (result != null) { return result; } @@ -335,12 +342,106 @@ public class IndexCacheOnDiscDeltaBased implements IndexCache { .create(); } + + /** + * just keep a md5 hash internally for identifying files to save memory + */ + private static class InternalFileIdentifier { + + public static InternalFileIdentifier fromPath(String fileName) { + byte[] id = DigestUtils.md5(fileName); + return new InternalFileIdentifier( id); + } + + private byte[] id; + + private InternalFileIdentifier(byte[] id) { + this.id = id; + } + + @Override + public int hashCode() { + return Arrays.hashCode(id); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + InternalFileIdentifier other = (InternalFileIdentifier) obj; + return Arrays.equals(id, other.id); + } + + } + + + + + /** + * internal storage structure + */ + private static class IndexCacheStore { + + @SuppressWarnings("unused") + private final String elementType; + + private final SortedMap timestampedFiles; + private final List elements; + private final Map> dependencies; + + public IndexCacheStore(SortedMap timestampedFiles, List elements, Map> dependencies, Class elementType) { + this.timestampedFiles = timestampedFiles; + this.elements = elements; + this.dependencies = dependencies; + this.elementType = elementType.getName(); + } + + public Map> getDependencies() { + return dependencies; + } + + public List getSymbols() { + return elements; + } + + public SortedMap getTimestampedFiles() { + return timestampedFiles; + } + + } + + + // + // + // internal delta-based storage structure: snapshots, updates, and deletions + // + // + + private static record DeltaStorage (DeltaElement storedElement) {} private static interface DeltaElement { public IndexCacheStore apply(IndexCacheStore store); } + private static class DeltaSnapshot implements DeltaElement { + + private IndexCacheStore store; + + public DeltaSnapshot(IndexCacheStore store) { + this.store = store; + } + + @Override + public IndexCacheStore apply(IndexCacheStore store) { + return this.store; + } + } + private static class DeltaDelete implements DeltaElement { private final String[] files; @@ -433,53 +534,12 @@ public class IndexCacheOnDiscDeltaBased implements IndexCache { } - private static class DeltaSnapshot implements DeltaElement { - - private IndexCacheStore store; - public DeltaSnapshot(IndexCacheStore store) { - this.store = store; - } - - @Override - public IndexCacheStore apply(IndexCacheStore store) { - return this.store; - } - } - - - /** - * internal storage structure - */ - private static class IndexCacheStore { - - @SuppressWarnings("unused") - private final String elementType; - - private final SortedMap timestampedFiles; - private final List elements; - private final Map> dependencies; - - public IndexCacheStore(SortedMap timestampedFiles, List elements, Map> dependencies, Class elementType) { - this.timestampedFiles = timestampedFiles; - this.elements = elements; - this.dependencies = dependencies; - this.elementType = elementType.getName(); - } - - public Map> getDependencies() { - return dependencies; - } - - public List getSymbols() { - return elements; - } - - public SortedMap getTimestampedFiles() { - return timestampedFiles; - } - - } + // + // + // GSON serialize / deserialize adapters for the various types involved here that have special needs around JSON + // + // private static class IndexCacheStoreAdapter implements JsonDeserializer> {