Spring/spring-ai
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove embedding from Document

Browse Source 
- This PR removes the reference of `embedding` from the `Document` object.
   - The Document is designed to contain the content and its metadata but not the embedding representation of its content. This provides clear separation of concern.
   - Prior to this change, all the vector stores are updated by not depending on the embedding from Document. Instead, use the embedding values returned by the Embedding Models directly: https://github.com/spring-projects/spring-ai/issues/1826

Resolves #1781
This commit is contained in:
Ilayaperumal Gopinathan
2024-12-18 18:33:41 +00:00
committed by Mark Pollack
parent bfc0b8285e
commit 0d122bb007
2 changed files with 8 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
spring-ai-core/src/main/java/org/springframework/ai/document/Document.java
View File

@@ -34,10 +34,9 @@ import org.springframework.util.StringUtils;
/**
* A document is a container for the content and metadata of a document. It also contains
* the document's unique ID and an optional embedding.
* the document's unique ID.
*
* A Document can hold either text content or media content, but not both. This ensures
* clear content type handling and processing.
* A Document can hold either text content or media content, but not both.
*
* It is intended to be used to take data from external sources as part of spring-ai's ETL
* pipeline and create an embedding for the text or media and store that embedding in a
@@ -122,12 +121,6 @@ public class Document {
@Nullable
private final Double score;
/**
* Embedding of the document. Note: ephemeral field.
*/
@JsonProperty(index = 100)
private float[] embedding = new float[0];
/**
* Mutable, ephemeral, content to text formatter. Defaults to Document text.
*/
@@ -260,22 +253,6 @@ public class Document {
return this.score;
}
/**
* Return the embedding that were calculated.
* @deprecated We are considering getting rid of this, please comment on
* https://github.com/spring-projects/spring-ai/issues/1781
* @return the embeddings
*/
@Deprecated(since = "1.0.0-M4")
public float[] getEmbedding() {
return this.embedding;
}
public void setEmbedding(float[] embedding) {
Assert.notNull(embedding, "embedding must not be null");
this.embedding = embedding;
}
/**
* Returns the content formatter associated with this document.
* @deprecated We are considering getting rid of this, please comment on
@@ -332,8 +309,6 @@ public class Document {
private Map<String, Object> metadata = new HashMap<>();
private float[] embedding = new float[0];
@Nullable
private Double score;
@@ -406,12 +381,6 @@ public class Document {
return this;
}
public Builder embedding(float[] embedding) {
Assert.notNull(embedding, "embedding cannot be null");
this.embedding = embedding;
return this;
}
/**
* Sets a score value for this document.
* <p>
@@ -436,9 +405,7 @@ public class Document {
if (!StringUtils.hasText(this.id)) {
this.id = this.idGenerator.generateId(this.text, this.metadata);
}
var document = new Document(this.id, this.text, this.media, this.metadata, this.score);
document.setEmbedding(this.embedding);
return document;
return new Document(this.id, this.text, this.media, this.metadata, this.score);
}
}

25
spring-ai-core/src/test/java/org/springframework/ai/document/DocumentTests.java
View File

@@ -16,18 +16,17 @@
package org.springframework.ai.document;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.springframework.ai.document.id.IdGenerator;
import org.springframework.ai.model.Media;
import org.springframework.util.MimeTypeUtils;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -211,20 +210,6 @@ public class DocumentTests {
assertThat(document.getMetadata()).containsEntry("key1", "value1").containsEntry("key2", "value2");
}
@Test
void testEmbeddingOperations() {
float[] embedding = new float[] { 0.1f, 0.2f, 0.3f };
Document document = Document.builder().text("test").embedding(embedding).build();
assertThat(document.getEmbedding()).isEqualTo(embedding);
}
@Test
void testNullEmbeddingThrowsException() {
assertThrows(IllegalArgumentException.class, () -> Document.builder().text("test").embedding(null).build());
}
private static Media getMedia() {
try {
URL mediaUrl1 = new URL("http://type1");