Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support similarity scores in Document API

Browse Source 
Document
* Introduced “score” attribute in Document API. It stores the similarity score.
* Consolidate “distance” metadata for Documents. It stores the distance measurement.
* Adopted prefix-less naming convention in Document.Builder and deprecated old methods.
* Deprecated the many overloaded Document constructors in favour of Document.Builder.

Vector Stores
* Every vector store implementation now configures a “score” attribute with the similarity score of the Document embedding. It also includes the “distance” metadata with the distance measurement.
* Fixed error in Elasticsearch where distance and similarity were mixed up.
* Added missing integration tests for SimpleVectorStore.
* The Azure Vector Store and HanaDB Vector Store do not include those measurements because the product documentation do not include information about how the similarity score is returned, and without access to the cloud products I could not verify that via debugging.
* Improved tests to actually assert the result of the similarity search based on the returned score.

Signed-off-by: Thomas Vitale <ThomasVitale@users.noreply.github.com>
This commit is contained in:
Thomas Vitale
2024-11-26 08:03:40 +01:00
committed by Mark Pollack
parent 50223d20e3
commit fe58fd30eb
59 changed files with 1096 additions and 434 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
document-readers/markdown-reader/src/main/java/org/springframework/ai/reader/markdown/MarkdownDocumentReader.java
View File

@@ -176,14 +176,14 @@ public class MarkdownDocumentReader implements DocumentReader {
}
translateLineBreakToSpace();
this.currentDocumentBuilder.withMetadata("category", "blockquote");
this.currentDocumentBuilder.metadata("category", "blockquote");
super.visit(blockQuote);
}
@Override
public void visit(Code code) {
this.currentParagraphs.add(code.getLiteral());
this.currentDocumentBuilder.withMetadata("category", "code_inline");
this.currentDocumentBuilder.metadata("category", "code_inline");
super.visit(code);
}
@@ -195,8 +195,8 @@ public class MarkdownDocumentReader implements DocumentReader {
translateLineBreakToSpace();
this.currentParagraphs.add(fencedCodeBlock.getLiteral());
this.currentDocumentBuilder.withMetadata("category", "code_block");
this.currentDocumentBuilder.withMetadata("lang", fencedCodeBlock.getInfo());
this.currentDocumentBuilder.metadata("category", "code_block");
this.currentDocumentBuilder.metadata("lang", fencedCodeBlock.getInfo());
buildAndFlush();
@@ -206,8 +206,8 @@ public class MarkdownDocumentReader implements DocumentReader {
@Override
public void visit(Text text) {
if (text.getParent() instanceof Heading heading) {
this.currentDocumentBuilder.withMetadata("category", "header_%d".formatted(heading.getLevel()))
.withMetadata("title", text.getLiteral());
this.currentDocumentBuilder.metadata("category", "header_%d".formatted(heading.getLevel()))
.metadata("title", text.getLiteral());
}
else {
this.currentParagraphs.add(text.getLiteral());
@@ -226,9 +226,9 @@ public class MarkdownDocumentReader implements DocumentReader {
if (!this.currentParagraphs.isEmpty()) {
String content = String.join("", this.currentParagraphs);
Document.Builder builder = this.currentDocumentBuilder.withContent(content);
Document.Builder builder = this.currentDocumentBuilder.content(content);
this.config.additionalMetadata.forEach(builder::withMetadata);
this.config.additionalMetadata.forEach(builder::metadata);
Document document = builder.build();